From 59cfe3dc8d6bc7b67cb19e300cc071050468eda5 Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Tue, 26 Feb 2019 11:07:47 -0500 Subject: [PATCH 01/85] SPARK-2067: Setup bash scripts for running system tests --- src/client/spark/Spark.csproj | 9 ++++++--- .../system_test_scripts/test_position_limits.sh | 3 +++ tools/run_sparkcmd_test_Linux.cmd | 5 ++++- 3 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 src/client/spark_test/system_test_scripts/test_position_limits.sh diff --git a/src/client/spark/Spark.csproj b/src/client/spark/Spark.csproj index c6eb7e982..626d1469c 100644 --- a/src/client/spark/Spark.csproj +++ b/src/client/spark/Spark.csproj @@ -17,9 +17,9 @@ bts 7.2 true - - 1.0.0 - + + 1.0.0 + @@ -90,6 +90,9 @@ MSBuild:Compile + + + diff --git a/src/client/spark_test/system_test_scripts/test_position_limits.sh b/src/client/spark_test/system_test_scripts/test_position_limits.sh new file mode 100644 index 000000000..899de5fd9 --- /dev/null +++ b/src/client/spark_test/system_test_scripts/test_position_limits.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +dotnet sparkcmd.dll test test_position_limits diff --git a/tools/run_sparkcmd_test_Linux.cmd b/tools/run_sparkcmd_test_Linux.cmd index c36fb6f69..77b96532d 100644 --- a/tools/run_sparkcmd_test_Linux.cmd +++ b/tools/run_sparkcmd_test_Linux.cmd @@ -16,7 +16,10 @@ dotnet publish -f netcoreapp2.1 cd ..\..\..\build\Debug\bin\netcoreapp2.1\publish bash -c "scp * bts@sparkvm:~/spark/build/ninja_%build_type%/server/" -ssh bts@sparkvm -p %SPARKVM_PORT% -o StrictHostKeyChecking=no -t "cd /home/bts/spark/build/ninja_%build_type%/server && dotnet sparkcmd.dll test %1 %2" +cd ../../net471 +bash -c "scp *.sh bts@sparkvm:~/spark/build/ninja_%build_type%/server/" + +ssh bts@sparkvm -p %SPARKVM_PORT% -o StrictHostKeyChecking=no -t "cd /home/bts/spark/build/ninja_%build_type%/server && ./%1 %2" ENDLOCAL popd From 7ed3a977ab1c60f1b8411fb2cb13fc5c4cadd640 Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Tue, 26 Feb 2019 11:18:26 -0500 Subject: [PATCH 02/85] SPARK-2065: Move AdminUtilities --- src/client/spark_test/System/LimitsTests.cs | 214 ---------------- src/client/spark_test/Utils/AdminUtilities.cs | 230 ++++++++++++++++++ 2 files changed, 230 insertions(+), 214 deletions(-) create mode 100644 src/client/spark_test/Utils/AdminUtilities.cs diff --git a/src/client/spark_test/System/LimitsTests.cs b/src/client/spark_test/System/LimitsTests.cs index 562234dd8..ba60f53c8 100644 --- a/src/client/spark_test/System/LimitsTests.cs +++ b/src/client/spark_test/System/LimitsTests.cs @@ -358,218 +358,4 @@ namespace bts.tests } - public class AdminUtilities { - - public static async Task get_account_data (AdminService admin_service, string account_name) - { - Update update = await admin_service.subscribei_admin_data().FirstAsync(); - - foreach (var acc in update.all.First().Accounts) { - if(acc.Name == account_name) { - return acc; - } - } - return null; - } - - public static async Task get_user_data (AdminService admin_service, string user_name) - { - Update update = await admin_service.subscribei_admin_data().FirstAsync(); - - foreach (var usr in update.all.First().Users) { - if(usr.Name == user_name) { - return usr; - } - } - return null; - } - - public static AccountSecurityClipLimitUnitImmutable? get_clip_limit_row(BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { - IEnumerable clip_limit_rows = account_data.Value.SecurityClipLimits.Limit.Where( - r => r.Security.SecuritySymbol == symbol - && r.Security.Exchange == venue - && r.Security.SecurityType == security_type); - return clip_limit_rows.Any() ? clip_limit_rows.First() : (AccountSecurityClipLimitUnitImmutable?) null; - } - - public static uint? get_clip_limit(BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { - var row = get_clip_limit_row(account_data,symbol,security_type,venue); - return row.HasValue ? row.Value.ClipSize : (uint?) null; - } - - public static async Task get_clip_limit(AdminService admin_service, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { - var account_data = await get_account_data(admin_service, account_name); - return get_clip_limit(account_data, symbol, security_type, venue); - } - - public static async Task set_clip_limit(AdminService admin_service, uint clip_value, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { - var account_data = await get_account_data(admin_service, account_name); - return await set_clip_limit(admin_service, clip_value,account_data, symbol,security_type, venue); - } - - public static async Task set_clip_limit(AdminService admin_service, uint clip_value, BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { - - AccountSecurityClipLimitUnitImmutable? origin_clip_limit_row = get_clip_limit_row(account_data,symbol,security_type,venue); - AccountSecurityClipLimitUnitImmutable new_clip_limit_row; - - List clip_limits_list = account_data.Value.SecurityClipLimits.Limit.ToList(); - - if(origin_clip_limit_row.HasValue) { - clip_limits_list.Remove(origin_clip_limit_row.Value); - } - - new_clip_limit_row = (AccountSecurityClipLimitUnitImmutable) new AccountSecurityClipLimitUnit { - Security = new SecurityScope(){ SecuritySymbol = symbol, Exchange = venue, SecurityType = security_type }, - ClipSize = clip_value - }; - - clip_limits_list.Add(new_clip_limit_row); - - AccountSecurityClipLimits temp_scl = new AccountSecurityClipLimits(); - temp_scl.Limit.AddRange(clip_limits_list.Select(x => x.CloneToMutable())); - - var new_account_data = account_data.Value.with(security_clip_limits: temp_scl); - - var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument { - CompleteData = new_account_data.CloneToMutable() - }); - - return update_ret.success; - } - - public static async Task clear_all_limits(AdminService admin_service, string account_name) { - var account_data = await get_account_data(admin_service, account_name); - return await clear_all_limits(admin_service, account_data); - } - - public static async Task clear_all_limits(AdminService admin_service, BTSAccountImmutable? account_data) { - var new_account_data = account_data.Value.with(security_clip_limits: new AccountSecurityClipLimits(), product_position_limits: new List()); - - var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument { - CompleteData = new_account_data.CloneToMutable() - }); - - return update_ret.success; - } - - - public static async Task remove_clip_limit(AdminService admin_service, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { - var account_data = await get_account_data(admin_service, account_name); - return await remove_clip_limit(admin_service, account_data, symbol, security_type, venue); - } - - public static async Task remove_clip_limit(AdminService admin_service, BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { - - AccountSecurityClipLimitUnitImmutable? origin_clip_limit_row = get_clip_limit_row(account_data,symbol,security_type,venue); - List clip_limits_list = account_data.Value.SecurityClipLimits.Limit.ToList(); - - if(origin_clip_limit_row.HasValue) { - clip_limits_list.Remove(origin_clip_limit_row.Value); - AccountSecurityClipLimits temp_scl = new AccountSecurityClipLimits(); - temp_scl.Limit.AddRange(clip_limits_list.Select(x => x.CloneToMutable())); - var new_account_data = account_data.Value.with(security_clip_limits: temp_scl); - - var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument { - CompleteData = new_account_data.CloneToMutable() - }); - - return update_ret.success; - } - else - { - return false; - } - } - - - public static async Task get_position_limit_row(AdminService admin_service, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { - var account_data = await get_account_data(admin_service, account_name); - return get_position_limit_row(account_data, symbol, security_type, venue); - } - - public static AccountProductPositionLimitUnitImmutable? get_position_limit_row(BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { - IEnumerable limit_rows = account_data.Value.ProductPositionLimits.Where( - r => r.Product.SecuritySymbol == symbol - && r.Product.Exchange == venue - && r.Product.SecurityType == security_type); - return limit_rows.Any() ? limit_rows.First() : (AccountProductPositionLimitUnitImmutable?) null; - - } - - public static async Task remove_position_limit(AdminService admin_service, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { - var account_data = await get_account_data(admin_service, account_name); - return await remove_position_limit(admin_service, account_data, symbol, security_type, venue); - } - - public static async Task remove_position_limit(AdminService admin_service, BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { - - AccountProductPositionLimitUnitImmutable? origin_limit_row = get_position_limit_row(account_data,symbol,security_type,venue); - List limits_list = account_data.Value.ProductPositionLimits.ToList(); - - if(origin_limit_row.HasValue) { - limits_list.Remove(origin_limit_row.Value); - List temp_plu = new List(); - temp_plu.AddRange(limits_list.Select(x => x.CloneToMutable())); - var new_account_data = account_data.Value.with(product_position_limits: temp_plu); - - var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument { - CompleteData = new_account_data.CloneToMutable() - }); - - return update_ret.success; - } - else - { - return false; - } - } - - public static async Task set_position_limit(AdminService admin_service, string account_name, string symbol, uint long_limit, uint short_limit, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { - var account_data = await get_account_data(admin_service, account_name); - return await set_position_limit(admin_service, account_data, symbol, long_limit, short_limit, security_type, venue); - } - - public static async Task set_position_limit(AdminService admin_service, BTSAccountImmutable? account_data, string symbol, uint long_limit, uint short_limit, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { - - AccountProductPositionLimitUnitImmutable? limit_row = get_position_limit_row(account_data,symbol,security_type,venue); - List limits_list = account_data.Value.ProductPositionLimits.ToList(); - - if(limit_row.HasValue) { - // update the row - limits_list.Remove(limit_row.Value); - limit_row = limit_row.Value.with(long_position_limit: long_limit, short_position_limit: short_limit); - } - else { - // create a new row - AccountProductPositionLimitUnit acplu = new AccountProductPositionLimitUnit(); - acplu.LongPositionLimit = long_limit; - acplu.ShortPositionLimit = short_limit; - acplu.Product = new ProductScope { - SecuritySymbol = symbol, - SecurityType = security_type, - Exchange = venue - }; - - limit_row = acplu; - } - - limits_list.Add(limit_row.Value); - - List temp_plu = new List(); - temp_plu.AddRange(limits_list.Select(x => x.CloneToMutable())); - - var new_account_data = account_data.Value.with(product_position_limits: temp_plu); - - var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument { - CompleteData = new_account_data.CloneToMutable() - }); - - return update_ret.success; - } - - - } - - - } diff --git a/src/client/spark_test/Utils/AdminUtilities.cs b/src/client/spark_test/Utils/AdminUtilities.cs new file mode 100644 index 000000000..d46160b16 --- /dev/null +++ b/src/client/spark_test/Utils/AdminUtilities.cs @@ -0,0 +1,230 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive.Linq; +using System.Text; +using System.Threading.Tasks; +using bts.api; +using bts.api.admin; +using bts.api.core; +using bts.api.test; +using bts.test; +using bts.api.orderman; +using mbts; +using mbts.tests; +using bts.co2; + +namespace bts.tests +{ + public class AdminUtilities { + + public static async Task get_account_data (AdminService admin_service, string account_name) + { + Update update = await admin_service.subscribei_admin_data().FirstAsync(); + + foreach (var acc in update.all.First().Accounts) { + if(acc.Name == account_name) { + return acc; + } + } + return null; + } + + public static async Task get_user_data (AdminService admin_service, string user_name) + { + Update update = await admin_service.subscribei_admin_data().FirstAsync(); + + foreach (var usr in update.all.First().Users) { + if(usr.Name == user_name) { + return usr; + } + } + return null; + } + + public static AccountSecurityClipLimitUnitImmutable? get_clip_limit_row(BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { + IEnumerable clip_limit_rows = account_data.Value.SecurityClipLimits.Limit.Where( + r => r.Security.SecuritySymbol == symbol + && r.Security.Exchange == venue + && r.Security.SecurityType == security_type); + return clip_limit_rows.Any() ? clip_limit_rows.First() : (AccountSecurityClipLimitUnitImmutable?) null; + } + + public static uint? get_clip_limit(BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { + var row = get_clip_limit_row(account_data,symbol,security_type,venue); + return row.HasValue ? row.Value.ClipSize : (uint?) null; + } + + public static async Task get_clip_limit(AdminService admin_service, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { + var account_data = await get_account_data(admin_service, account_name); + return get_clip_limit(account_data, symbol, security_type, venue); + } + + public static async Task set_clip_limit(AdminService admin_service, uint clip_value, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { + var account_data = await get_account_data(admin_service, account_name); + return await set_clip_limit(admin_service, clip_value,account_data, symbol,security_type, venue); + } + + public static async Task set_clip_limit(AdminService admin_service, uint clip_value, BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { + + AccountSecurityClipLimitUnitImmutable? origin_clip_limit_row = get_clip_limit_row(account_data,symbol,security_type,venue); + AccountSecurityClipLimitUnitImmutable new_clip_limit_row; + + List clip_limits_list = account_data.Value.SecurityClipLimits.Limit.ToList(); + + if(origin_clip_limit_row.HasValue) { + clip_limits_list.Remove(origin_clip_limit_row.Value); + } + + new_clip_limit_row = (AccountSecurityClipLimitUnitImmutable) new AccountSecurityClipLimitUnit { + Security = new SecurityScope(){ SecuritySymbol = symbol, Exchange = venue, SecurityType = security_type }, + ClipSize = clip_value + }; + + clip_limits_list.Add(new_clip_limit_row); + + AccountSecurityClipLimits temp_scl = new AccountSecurityClipLimits(); + temp_scl.Limit.AddRange(clip_limits_list.Select(x => x.CloneToMutable())); + + var new_account_data = account_data.Value.with(security_clip_limits: temp_scl); + + var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument { + CompleteData = new_account_data.CloneToMutable() + }); + + return update_ret.success; + } + + public static async Task clear_all_limits(AdminService admin_service, string account_name) { + var account_data = await get_account_data(admin_service, account_name); + return await clear_all_limits(admin_service, account_data); + } + + public static async Task clear_all_limits(AdminService admin_service, BTSAccountImmutable? account_data) { + var new_account_data = account_data.Value.with(security_clip_limits: new AccountSecurityClipLimits(), product_position_limits: new List()); + + var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument { + CompleteData = new_account_data.CloneToMutable() + }); + + return update_ret.success; + } + + + public static async Task remove_clip_limit(AdminService admin_service, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { + var account_data = await get_account_data(admin_service, account_name); + return await remove_clip_limit(admin_service, account_data, symbol, security_type, venue); + } + + public static async Task remove_clip_limit(AdminService admin_service, BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { + + AccountSecurityClipLimitUnitImmutable? origin_clip_limit_row = get_clip_limit_row(account_data,symbol,security_type,venue); + List clip_limits_list = account_data.Value.SecurityClipLimits.Limit.ToList(); + + if(origin_clip_limit_row.HasValue) { + clip_limits_list.Remove(origin_clip_limit_row.Value); + AccountSecurityClipLimits temp_scl = new AccountSecurityClipLimits(); + temp_scl.Limit.AddRange(clip_limits_list.Select(x => x.CloneToMutable())); + var new_account_data = account_data.Value.with(security_clip_limits: temp_scl); + + var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument { + CompleteData = new_account_data.CloneToMutable() + }); + + return update_ret.success; + } + else + { + return false; + } + } + + + public static async Task get_position_limit_row(AdminService admin_service, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { + var account_data = await get_account_data(admin_service, account_name); + return get_position_limit_row(account_data, symbol, security_type, venue); + } + + public static AccountProductPositionLimitUnitImmutable? get_position_limit_row(BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { + IEnumerable limit_rows = account_data.Value.ProductPositionLimits.Where( + r => r.Product.SecuritySymbol == symbol + && r.Product.Exchange == venue + && r.Product.SecurityType == security_type); + return limit_rows.Any() ? limit_rows.First() : (AccountProductPositionLimitUnitImmutable?) null; + + } + + public static async Task remove_position_limit(AdminService admin_service, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { + var account_data = await get_account_data(admin_service, account_name); + return await remove_position_limit(admin_service, account_data, symbol, security_type, venue); + } + + public static async Task remove_position_limit(AdminService admin_service, BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { + + AccountProductPositionLimitUnitImmutable? origin_limit_row = get_position_limit_row(account_data,symbol,security_type,venue); + List limits_list = account_data.Value.ProductPositionLimits.ToList(); + + if(origin_limit_row.HasValue) { + limits_list.Remove(origin_limit_row.Value); + List temp_plu = new List(); + temp_plu.AddRange(limits_list.Select(x => x.CloneToMutable())); + var new_account_data = account_data.Value.with(product_position_limits: temp_plu); + + var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument { + CompleteData = new_account_data.CloneToMutable() + }); + + return update_ret.success; + } + else + { + return false; + } + } + + public static async Task set_position_limit(AdminService admin_service, string account_name, string symbol, uint long_limit, uint short_limit, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { + var account_data = await get_account_data(admin_service, account_name); + return await set_position_limit(admin_service, account_data, symbol, long_limit, short_limit, security_type, venue); + } + + public static async Task set_position_limit(AdminService admin_service, BTSAccountImmutable? account_data, string symbol, uint long_limit, uint short_limit, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) { + + AccountProductPositionLimitUnitImmutable? limit_row = get_position_limit_row(account_data,symbol,security_type,venue); + List limits_list = account_data.Value.ProductPositionLimits.ToList(); + + if(limit_row.HasValue) { + // update the row + limits_list.Remove(limit_row.Value); + limit_row = limit_row.Value.with(long_position_limit: long_limit, short_position_limit: short_limit); + } + else { + // create a new row + AccountProductPositionLimitUnit acplu = new AccountProductPositionLimitUnit(); + acplu.LongPositionLimit = long_limit; + acplu.ShortPositionLimit = short_limit; + acplu.Product = new ProductScope { + SecuritySymbol = symbol, + SecurityType = security_type, + Exchange = venue + }; + + limit_row = acplu; + } + + limits_list.Add(limit_row.Value); + + List temp_plu = new List(); + temp_plu.AddRange(limits_list.Select(x => x.CloneToMutable())); + + var new_account_data = account_data.Value.with(product_position_limits: temp_plu); + + var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument { + CompleteData = new_account_data.CloneToMutable() + }); + + return update_ret.success; + } + + + } +} From 482ca6b4845d2df992a50d4f49d5cfb5407e6ead Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Tue, 26 Feb 2019 14:02:09 -0500 Subject: [PATCH 03/85] SPARK-2065: Move order functions to OrderUtilities --- src/client/spark_test/System/LimitsTests.cs | 58 +++++-------------- src/client/spark_test/Utils/OrderUtilities.cs | 48 +++++++++++++++ 2 files changed, 64 insertions(+), 42 deletions(-) create mode 100644 src/client/spark_test/Utils/OrderUtilities.cs diff --git a/src/client/spark_test/System/LimitsTests.cs b/src/client/spark_test/System/LimitsTests.cs index ba60f53c8..eb917e950 100644 --- a/src/client/spark_test/System/LimitsTests.cs +++ b/src/client/spark_test/System/LimitsTests.cs @@ -43,6 +43,7 @@ namespace bts.tests Disposable.dispose(ref _order_service); Disposable.dispose(ref _admin_service); Disposable.dispose(ref _context); + Disposable.dispose(ref _order_subscription); } async Task> all_futures(string symbol) { @@ -109,7 +110,7 @@ namespace bts.tests var second_order_def = create_order_def(second_sec.Id, Side.Buy, test_account, test_user); - Result no_limit_order_result = await send_order(first_order_def, 5, 1); + Result no_limit_order_result = await OrderUtilities.send_order(_order_service, first_order_def, 5, 1); IEnumerable no_limit_filtered_orders = await get_filtered_orders(x => x.Key == no_limit_order_result.value.OrderKey); @@ -121,11 +122,11 @@ namespace bts.tests List> orders_response_list = new List>(); for(int i = 0; i <3; i++){ - orders_response_list.Add(await send_order(first_order_def, 5, 10)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, first_order_def, 5, 10)); } for(int i = 0; i <3; i++){ - orders_response_list.Add(await send_order(second_order_def, 5, 10)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, second_order_def, 5, 10)); } foreach(var i in orders_response_list){ @@ -133,24 +134,24 @@ namespace bts.tests Assert.IsTrue(iret.First().State.Status == OrderStatus.Working, "This order should have been 'working'"); } - var fail_order_result = await send_order(first_order_def, 5, 1); + var fail_order_result = await OrderUtilities.send_order(_order_service, first_order_def, 5, 1); var fail_order_filtered_orders = await get_filtered_orders(x => x.Key == fail_order_result.value.OrderKey); Assert.IsTrue(fail_order_filtered_orders.First().State.Status == OrderStatus.RejectedBySpark, "This order should have been 'Rejected by Spark'"); bool position_increase_success = await AdminUtilities.set_position_limit(_admin_service, test_account.name, test_symbol, 70, 120); Assert.IsTrue(position_increase_success); - var increased_order_result = await send_order(first_order_def, 5, 10); + var increased_order_result = await OrderUtilities.send_order(_order_service, first_order_def, 5, 10); var increased_order_filtered_orders = await get_filtered_orders(x => x.Key == increased_order_result.value.OrderKey); Assert.IsTrue(increased_order_filtered_orders.First().State.Status == OrderStatus.Working, "This order should have been 'Working'"); - var second_fail_order_result = await send_order(first_order_def, 5, 10); + var second_fail_order_result = await OrderUtilities.send_order(_order_service, first_order_def, 5, 10); var second_fail_filtered_orders = await get_filtered_orders(x => x.Key == second_fail_order_result.value.OrderKey); Assert.IsTrue(second_fail_filtered_orders.First().State.Status == OrderStatus.RejectedBySpark, "This order should have been 'Rejected by Spark'"); /* Test Short Position Limit */ - Assert.IsTrue(await cancel_orders(), "Unable to cancel orders"); + Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, _orders.all), "Unable to cancel orders"); await AdminUtilities.remove_position_limit(_admin_service, test_account.name, test_symbol); @@ -162,7 +163,7 @@ namespace bts.tests uint short_no_limit_order_key; - Result short_no_limit_order_result = await send_order(first_short_order_def, 50000, 7); + Result short_no_limit_order_result = await OrderUtilities.send_order(_order_service, first_short_order_def, 50000, 7); short_no_limit_order_key = short_no_limit_order_result.value.OrderKey; IEnumerable short_no_limit_filtered_orders = await get_filtered_orders(x => x.Key == short_no_limit_order_key); @@ -175,11 +176,11 @@ namespace bts.tests List> short_orders_response_list = new List>(); for(int i = 0; i <3; i++){ - short_orders_response_list.Add(await send_order(first_short_order_def, 50000, 10)); + short_orders_response_list.Add(await OrderUtilities.send_order(_order_service, first_short_order_def, 50000, 10)); } for(int i = 0; i <3; i++){ - short_orders_response_list.Add(await send_order(second_short_order_def, 50000, 10)); + short_orders_response_list.Add(await OrderUtilities.send_order(_order_service, second_short_order_def, 50000, 10)); } foreach(var i in short_orders_response_list){ @@ -187,22 +188,22 @@ namespace bts.tests Assert.IsTrue(iret.First().State.Status == OrderStatus.Working, "This order should have been 'working'"); } - var short_fail_order_result = await send_order(first_short_order_def, 50000, 1); + var short_fail_order_result = await OrderUtilities.send_order(_order_service, first_short_order_def, 50000, 1); var short_fail_order_filtered_orders = await get_filtered_orders(x => x.Key == short_fail_order_result.value.OrderKey); Assert.IsTrue(short_fail_order_filtered_orders.First().State.Status == OrderStatus.RejectedBySpark, "This order should have been 'Rejected by Spark'"); bool short_position_increase_success = await AdminUtilities.set_position_limit(_admin_service, test_account.name, test_symbol, 120, 70); Assert.IsTrue(short_position_increase_success); - var short_increased_order_result = await send_order(first_short_order_def, 50000, 10); + var short_increased_order_result = await OrderUtilities.send_order(_order_service, first_short_order_def, 50000, 10); var short_increased_order_filtered_orders = await get_filtered_orders(x => x.Key == short_increased_order_result.value.OrderKey); Assert.IsTrue(short_increased_order_filtered_orders.First().State.Status == OrderStatus.Working, "This order should have been 'Working'"); - var short_second_fail_order_result = await send_order(first_short_order_def, 50000, 10); + var short_second_fail_order_result = await OrderUtilities.send_order(_order_service, first_short_order_def, 50000, 10); var short_second_fail_filtered_orders = await get_filtered_orders(x => x.Key == short_second_fail_order_result.value.OrderKey); Assert.IsTrue(short_second_fail_filtered_orders.First().State.Status == OrderStatus.RejectedBySpark, "This order should have been 'Rejected by Spark'"); - Assert.IsTrue(await cancel_orders(), "Unable to cancel orders"); + Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, _orders.all), "Unable to cancel orders"); } @@ -286,6 +287,7 @@ namespace bts.tests } Assert.IsTrue(success, "Failed to restore Limit"); + Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, _orders.all), "Unable to cancel orders"); } @@ -311,34 +313,6 @@ namespace bts.tests } - - public async Task> send_order(OrderDefinition order_def, double price, uint size){ - return await _order_service.create_order_async(new Create() { - NewOrder = order_def, - Price = price, - Quantity = size - }); - } - - public async Task cancel_orders(uint key = 0){ - if (key > 0) { - Result resp = await _order_service.cancel_order_async(new Cancel() { OrderKey = key }); - return resp.success; - } - else { - bool suc = true; - var working_orders = await get_filtered_orders(x => x.State.Status == OrderStatus.Working); - if(working_orders != null){ - foreach (var o in working_orders){ - Result resp = await _order_service.cancel_order_async(new Cancel() { OrderKey = o.Key }); - suc &= resp.success; - } - return suc; - } - return true;// there are no working orders... so true enough - } - } - public OrderDefinition create_order_def(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user){ return new OrderDefinition { SecurityKey = security_id, diff --git a/src/client/spark_test/Utils/OrderUtilities.cs b/src/client/spark_test/Utils/OrderUtilities.cs new file mode 100644 index 000000000..5f22667bf --- /dev/null +++ b/src/client/spark_test/Utils/OrderUtilities.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive.Linq; +using System.Text; +using System.Threading.Tasks; +using bts.api; +using bts.api.admin; +using bts.api.core; +using bts.api.test; +using bts.test; +using bts.api.orderman; +using mbts; +using mbts.tests; +using bts.co2; + +namespace bts.tests +{ + public class OrderUtilities { + public static async Task> send_order(OrderService order_service, OrderDefinition order_def, double price, uint size){ + return await order_service.create_order_async(new Create() { + NewOrder = order_def, + Price = price, + Quantity = size + }); + } + + public static async Task cancel_orders(OrderService order_service, IEnumerable orders, uint key = 0){ + if (key > 0) { + Result resp = await order_service.cancel_order_async(new Cancel() { OrderKey = key }); + return resp.success; + } + else { + bool suc = true; + var working_orders = orders.Where(x => x.State.Status == OrderStatus.Working); + if(working_orders != null){ + foreach (var o in working_orders){ + Result resp = await order_service.cancel_order_async(new Cancel() { OrderKey = o.Key }); + suc &= resp.success; + } + return suc; + } + return true;// there are no working orders... so true enough + } + } + + } +} \ No newline at end of file From b7fcd17ef098a8f404d8d0fa12c4a7aaf3f790a1 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Wed, 27 Feb 2019 11:05:23 -0600 Subject: [PATCH 04/85] first test; create retrieve update delete of algo definitions --- .../spark_test/System/AlgoDefinitionTests.cs | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 src/client/spark_test/System/AlgoDefinitionTests.cs diff --git a/src/client/spark_test/System/AlgoDefinitionTests.cs b/src/client/spark_test/System/AlgoDefinitionTests.cs new file mode 100644 index 000000000..26673c7d7 --- /dev/null +++ b/src/client/spark_test/System/AlgoDefinitionTests.cs @@ -0,0 +1,132 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive.Linq; +using System.Text; +using System.Threading.Tasks; +using bts.api; +using bts.api.admin; +using bts.api.core; +using bts.api.test; +using bts.test; +using bts.api.orderman; +using mbts; +using mbts.tests; +using bts.co2; +using bts.api.orderman.algo; + +namespace bts.tests { + [TestGroup(TestType = TestType.System)] + public class AlgoDefinitionTests : IDisposable { + List _resources; + ServiceContext _context; + //OrderService _order_service; + AdminService _admin_service; + AlgoDefinitionService _algo_definition_service; + IEnumerable _non_deleted_defs; + Random _rng = new Random(); + + public AlgoDefinitionTests() { + _context = new ServiceContext(); + //_order_service = _context.locate(); + _admin_service = _context.locate(); + _algo_definition_service = _context.locate(); + + _resources = new List { + _context, + _admin_service, + _algo_definition_service, + _algo_definition_service.subscribei_algo_definitions().Subscribe(defs => _non_deleted_defs = defs.all.Where( x => x.Metadata.IsActive)) + }; + + var auth = _context.login("bts", ""); + auth.Wait(3000); + } + + [Test(Timeout = 20000)] + public async Task test_crud_algo_definitions() { + var created_algo_records = new List(); + var watch = new System.Diagnostics.Stopwatch(); + watch.Start(); + var user_data = await AdminUtilities.get_user_data(_admin_service, "bts"); + int num_created = 0; + if (user_data.HasValue) { + // create 100 algo defs + while (num_created < 100) { + var result = await _algo_definition_service.save_algo_definition_async( + new AlgoDefinitionUpdateRequest { + UserId = user_data.Value.Id, + Key = 0, + Algo = generate_random_algo_definition() + } + ); + Assert.IsTrue(result.value.Success, "There was a problem creating an algo defintion"); + created_algo_records.Add(result.value.Record); + num_created++; + } + + // update the 100 we just created algo defs + foreach(var r in created_algo_records) { + var def = r.Algo; + def.Name += " modified"; + await _algo_definition_service.save_algo_definition_async( + new AlgoDefinitionUpdateRequest { + UserId = user_data.Value.Id, + Key = r.Key, + Algo = def + } + ); + } + + // delete all non-deleted algo defs + Assert.IsTrue(_non_deleted_defs.Count() > 0, "There are no algo definitions to delete."); + foreach (var i in _non_deleted_defs.Select(x => x.Key).ToList()) { + var result = await _algo_definition_service.remove_algo_definition_async(new AlgoDefinitionRemoveRequest { Key = i }); + Assert.IsTrue(result.value.Success, "There was a problem removing an algo definition"); + } + Assert.IsTrue(_non_deleted_defs.Count() == 0, "There was a problem removing all algo definitions"); + + watch.Stop(); + Console.WriteLine($"Execution Time: {watch.ElapsedMilliseconds} ms"); + } + } + + // not quite random yet + private AlgoDefinition generate_random_algo_definition() { + return new AlgoDefinition { + Definition = new CustomOrderDefinition { + Iceberg = new IcebergOrderDefinition { + ShowQuantity = _rng.NextDouble() * 100, + Child = new CustomOrderDefinition { + Limit = new LimitOrderDefinition { } + } + }, + }, + Name = $"Test Algo Def: {generate_name(_rng, _rng.Next(4, 15))}" + }; + } + + // https://stackoverflow.com/questions/14687658/random-name-generator-in-c-sharp + public static string generate_name(Random r, int len) { + string[] consonants = { "b", "c", "d", "f", "g", "h", "j", "k", "l", "m", "l", "n", "p", "q", "r", "s", "sh", "zh", "t", "v", "w", "x" }; + string[] vowels = { "a", "e", "i", "o", "u", "ae", "y" }; + string name = ""; + name += consonants[r.Next(consonants.Length)].ToUpper(); + name += vowels[r.Next(vowels.Length)]; + int b = 2; //b tells how many times a new letter has been added. It's 2 right now because the first two letters are already in the name. + while (b < len) { + name += consonants[r.Next(consonants.Length)]; + b++; + name += vowels[r.Next(vowels.Length)]; + b++; + } + return name; + } + + public void Dispose() { + Disposable.dispose(_resources); + } + + } + +} From 6fb8fdc9f6acb956530088be4fb534ff2ae06087 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Wed, 27 Feb 2019 14:28:45 -0600 Subject: [PATCH 05/85] updating test timeout --- src/client/spark_test/System/AlgoDefinitionTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark_test/System/AlgoDefinitionTests.cs b/src/client/spark_test/System/AlgoDefinitionTests.cs index 26673c7d7..3268a5a72 100644 --- a/src/client/spark_test/System/AlgoDefinitionTests.cs +++ b/src/client/spark_test/System/AlgoDefinitionTests.cs @@ -43,7 +43,7 @@ namespace bts.tests { auth.Wait(3000); } - [Test(Timeout = 20000)] + [Test(Timeout = 25)] public async Task test_crud_algo_definitions() { var created_algo_records = new List(); var watch = new System.Diagnostics.Stopwatch(); From 0caa6b6ee0622e189af3ead9733f73937320d723 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Fri, 8 Mar 2019 08:53:00 -0600 Subject: [PATCH 06/85] IB test 1 --- .../spark_test/System/IcebergAlgoTests.cs | 219 ++++++++++++++++++ 1 file changed, 219 insertions(+) create mode 100644 src/client/spark_test/System/IcebergAlgoTests.cs diff --git a/src/client/spark_test/System/IcebergAlgoTests.cs b/src/client/spark_test/System/IcebergAlgoTests.cs new file mode 100644 index 000000000..1680654ae --- /dev/null +++ b/src/client/spark_test/System/IcebergAlgoTests.cs @@ -0,0 +1,219 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive.Linq; +using System.Text; +using System.Threading.Tasks; +using bts.api; +using bts.api.admin; +using bts.api.core; +using bts.api.test; +using bts.test; +using bts.api.orderman; +using mbts; +using mbts.tests; +using bts.co2; +using bts.api.orderman.algo; + +namespace bts.tests { + + static class TestUtils { + public static bool test_price_equality(double price_1, double price_2, double tick_size) { + return Math.Abs(price_2 - price_1) < 0.01 * tick_size; + } + + // this is a copy from TickUtils -- ask what's legal for the spark_test project... + public static double ticksize(this ContractSpecificationImmutable spec, double price) => + Math.Abs(price) > spec.PriceBreak ? spec.HighTick : spec.LowTick; + + } + + + [TestGroup(TestType = TestType.System)] + class IcebergAlgoTests : IDisposable { + List _resources; + ServiceContext _context; + AdminService _admin_service; + OrderService _order_service; + Update _orders; + Update _fills; + + public IcebergAlgoTests() { + _context = new ServiceContext(); + _admin_service = _context.locate(); + _order_service = _context.locate(); + + _resources = new List { + _context, + _admin_service, + _order_service, + _order_service.subscribei_orders().Subscribe(ords => { + _orders = ords; + }), + _order_service.subscribei_fills().Subscribe(phills => + { + // lol >8) + _fills = phills; + }) + }; + + var auth = _context.login("bts", ""); + auth.Wait(3000); + } + + + async Task get_security_with_no_market(string symbol) { + var request = new MDRequest() { + Host = "system_test", + Channels = { + new SecurityChannel() { + Time = 0, + Flags = (uint)MDRequest.Types.Flags.Bbo, + Type = SecurityType.Future, + Exchange = Venue.Sim, + Symbol = symbol, + TradeCursor = 0, + RfqCursor = 0 + } + } + }; + + var relay = _context.locate(); + await Task.Delay(100); + var result = await relay.requestmd_async(request); + + SecurityImmutable? security_with_no_market = new SecurityImmutable?(); + if (result.success) { + foreach (var ch in result.value.Channels) { + foreach (var update in ch.Updates) { + if (update.Direct.Asks.Count == 0 && update.Direct.Bids.Count == 0) { + security_with_no_market = new SecurityImmutable(update.Security); + break; + } + } + if (security_with_no_market.HasValue) { + break; + } + } + } + return security_with_no_market; + } + + public OrderDefinition create_order_def(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user) { + return new OrderDefinition { + SecurityKey = security_id, + Side = side, + TimeInForce = TimeInForce.Day, + CustomDef = new CustomOrderDefinition { Limit = new LimitOrderDefinition { } }, + CtiCode = CTICode.CtiCode1, + CustomerOrFirm = CustomerOrFirm.Firm, + ManualOrderIndicator = ManualOrderIndicator.Yes, + Account = account.name, + AccountId = account.id, + User = user.name, + UserId = user.id + }; + } + + public OrderDefinition create_iceberg_order_def(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user) { + return new OrderDefinition { + SecurityKey = security_id, + Side = side, + TimeInForce = TimeInForce.Day, + CustomDef = new CustomOrderDefinition { + Iceberg = new IcebergOrderDefinition { + ShowQuantity = 3, + Child = new CustomOrderDefinition { + Limit = new LimitOrderDefinition { } + } + } + }, + CtiCode = CTICode.CtiCode1, + CustomerOrFirm = CustomerOrFirm.Firm, + ManualOrderIndicator = ManualOrderIndicator.Yes, + Account = account.name, + AccountId = account.id, + User = user.name, + UserId = user.id + }; + } + + private async Task> get_filtered_orders(Func predicate, int count = 1) { + int current_count = 0; + int it = 0; + do { + if (_orders != null) { + var ords = _orders.all.Where(predicate); + current_count = ords.Count(); + + if (current_count >= count) { + return ords; + } + it++; + } + await Task.Delay(100); + } + while (it < 100 && (_orders == null || current_count < count)); + + return null; + + } + + [Test(Timeout = 25)] + public async Task test_ib_create_and_takeout_market() { + var test_user = (name: "bts", id: 0u); + var test_account = (name: "ACC A", id: 0u); + + // get the ACC A account + BTSAccountImmutable? account_data = await AdminUtilities.get_account_data(_admin_service, test_account.name); + Assert.IsTrue(account_data.HasValue); + test_account.id = account_data.Value.Id; + + // get the bts user + var user_data = await AdminUtilities.get_user_data(_admin_service, test_user.name); + Assert.IsTrue(user_data.HasValue); + test_user.id = user_data.Value.Id; + + // find a security with no market + var s = await get_security_with_no_market("CL"); + Assert.IsTrue(s.HasValue, "Could not find a security with no maket"); + + // create a market in the instrument + List> orders_response_list = new List>(); + var bid_order_def = create_order_def(s.Value.Id, Side.Buy, test_account, test_user); + var ask_order_def = create_order_def(s.Value.Id, Side.Sell, test_account, test_user); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 55.00, 50)); + orders_response_list.Add( await OrderUtilities.send_order(_order_service, ask_order_def, 55.05, 50)); + List test_orders = new List(); + foreach (var i in orders_response_list) { + var iret = await get_filtered_orders(x => x.Key == i.value.OrderKey); + Assert.IsTrue(iret.First().State.Status == OrderStatus.Working, "This order should have been 'working'"); + test_orders.AddRange(iret); + } + + // create an IB that will cross this market + var ib_order_def = create_iceberg_order_def(s.Value.Id, Side.Buy, test_account, test_user); + var ib_order_result = await OrderUtilities.send_order(_order_service, ib_order_def, 55.05, 30); + await Task.Delay(new TimeSpan(0, 0, 3)); + var ib_filtered_order = await get_filtered_orders(x => x.Key == ib_order_result.value.OrderKey); + var ib_child_orders = await get_filtered_orders(x => x.ParentKey == ib_order_result.value.OrderKey, 10); + + Assert.IsTrue(ib_filtered_order.First().State.Status == OrderStatus.Completed, "This IB order should have been 'Completed'"); + Assert.IsTrue(ib_child_orders.Count() == 10, "There should have been 10 child orders"); + Assert.IsTrue(ib_child_orders.All(x => x.HasState && x.State.Status == OrderStatus.Completed), "All child orders should be completed"); + + var filled_childen = _fills.all.Where(x => ib_child_orders.Select(o => o.Key).Contains(x.OrderKey)); + Assert.IsTrue(filled_childen.Count() == 10, "There should have been 10 fills"); + Assert.IsTrue(filled_childen.All(x => + TestUtils.test_price_equality(x.LastPrice, 55.05, s.Value.ContractSpec.ticksize(x.LastPrice)) && x.LastShares == 3), "All child fills should have been filled at price 55.50 with quantity 3"); + + // cleanup + Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, test_orders), "Unable to cancel test orders"); + } + + public void Dispose() { + Disposable.dispose(_resources); + } + + } +} From f7cdb1f00bf2ee6d16bdb2cc8518fd811bdf4102 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Fri, 8 Mar 2019 10:36:51 -0600 Subject: [PATCH 07/85] refactor/cleanup of some common functions --- .../spark_test/System/IcebergAlgoTests.cs | 57 ++++--------------- src/client/spark_test/Utils/OrderUtilities.cs | 41 +++++++++++++ 2 files changed, 51 insertions(+), 47 deletions(-) diff --git a/src/client/spark_test/System/IcebergAlgoTests.cs b/src/client/spark_test/System/IcebergAlgoTests.cs index 1680654ae..e91ca574f 100644 --- a/src/client/spark_test/System/IcebergAlgoTests.cs +++ b/src/client/spark_test/System/IcebergAlgoTests.cs @@ -13,7 +13,6 @@ using bts.api.orderman; using mbts; using mbts.tests; using bts.co2; -using bts.api.orderman.algo; namespace bts.tests { @@ -99,22 +98,6 @@ namespace bts.tests { return security_with_no_market; } - public OrderDefinition create_order_def(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user) { - return new OrderDefinition { - SecurityKey = security_id, - Side = side, - TimeInForce = TimeInForce.Day, - CustomDef = new CustomOrderDefinition { Limit = new LimitOrderDefinition { } }, - CtiCode = CTICode.CtiCode1, - CustomerOrFirm = CustomerOrFirm.Firm, - ManualOrderIndicator = ManualOrderIndicator.Yes, - Account = account.name, - AccountId = account.id, - User = user.name, - UserId = user.id - }; - } - public OrderDefinition create_iceberg_order_def(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user) { return new OrderDefinition { SecurityKey = security_id, @@ -138,27 +121,6 @@ namespace bts.tests { }; } - private async Task> get_filtered_orders(Func predicate, int count = 1) { - int current_count = 0; - int it = 0; - do { - if (_orders != null) { - var ords = _orders.all.Where(predicate); - current_count = ords.Count(); - - if (current_count >= count) { - return ords; - } - it++; - } - await Task.Delay(100); - } - while (it < 100 && (_orders == null || current_count < count)); - - return null; - - } - [Test(Timeout = 25)] public async Task test_ib_create_and_takeout_market() { var test_user = (name: "bts", id: 0u); @@ -180,25 +142,26 @@ namespace bts.tests { // create a market in the instrument List> orders_response_list = new List>(); - var bid_order_def = create_order_def(s.Value.Id, Side.Buy, test_account, test_user); - var ask_order_def = create_order_def(s.Value.Id, Side.Sell, test_account, test_user); + var bid_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Buy, test_account, test_user); + var ask_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Sell, test_account, test_user); orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 55.00, 50)); - orders_response_list.Add( await OrderUtilities.send_order(_order_service, ask_order_def, 55.05, 50)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, 55.05, 50)); + await Task.Delay(new TimeSpan(0, 0, 3)); List test_orders = new List(); foreach (var i in orders_response_list) { - var iret = await get_filtered_orders(x => x.Key == i.value.OrderKey); - Assert.IsTrue(iret.First().State.Status == OrderStatus.Working, "This order should have been 'working'"); + var iret = await OrderUtilities.get_filtered_orders(_orders, x => x.Key == i.value.OrderKey); + Assert.IsTrue(iret.Count() > 0 && iret.First().State.Status == OrderStatus.Working, "This order should have been 'working'"); test_orders.AddRange(iret); } // create an IB that will cross this market var ib_order_def = create_iceberg_order_def(s.Value.Id, Side.Buy, test_account, test_user); var ib_order_result = await OrderUtilities.send_order(_order_service, ib_order_def, 55.05, 30); - await Task.Delay(new TimeSpan(0, 0, 3)); - var ib_filtered_order = await get_filtered_orders(x => x.Key == ib_order_result.value.OrderKey); - var ib_child_orders = await get_filtered_orders(x => x.ParentKey == ib_order_result.value.OrderKey, 10); + await Task.Delay(new TimeSpan(0, 0, 6)); + var ib_filtered_order = await OrderUtilities.get_filtered_orders(_orders, x => x.Key == ib_order_result.value.OrderKey); + var ib_child_orders = await OrderUtilities.get_filtered_orders(_orders, x => x.ParentKey == ib_order_result.value.OrderKey, 10); - Assert.IsTrue(ib_filtered_order.First().State.Status == OrderStatus.Completed, "This IB order should have been 'Completed'"); + Assert.IsTrue(ib_filtered_order.Count() > 0 && ib_filtered_order.First().State.Status == OrderStatus.Completed, "This IB order should have been 'Completed'"); Assert.IsTrue(ib_child_orders.Count() == 10, "There should have been 10 child orders"); Assert.IsTrue(ib_child_orders.All(x => x.HasState && x.State.Status == OrderStatus.Completed), "All child orders should be completed"); diff --git a/src/client/spark_test/Utils/OrderUtilities.cs b/src/client/spark_test/Utils/OrderUtilities.cs index 5f22667bf..fe099deba 100644 --- a/src/client/spark_test/Utils/OrderUtilities.cs +++ b/src/client/spark_test/Utils/OrderUtilities.cs @@ -44,5 +44,46 @@ namespace bts.tests } } + public static async Task> get_filtered_orders(Update orders, Func predicate, int count = 1) { + int current_count = 0; + int it = 0; + do { + if (orders != null) { + var ords = orders.all.Where(predicate); + current_count = ords.Count(); + + if (current_count >= count) { + return ords; + } + it++; + } + await Task.Delay(100); + } + while (it < 100 && (orders == null || current_count < count)); + + return new List(); + } + + } + + /// + /// We should replace with something like the OrderDataHeler from the Spark project. + /// + public static class OrderDefinitionUtilities { + public static OrderDefinition create_limit_order_def(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user) { + return new OrderDefinition { + SecurityKey = security_id, + Side = side, + TimeInForce = TimeInForce.Day, + CustomDef = new CustomOrderDefinition { Limit = new LimitOrderDefinition { } }, + CtiCode = CTICode.CtiCode1, + CustomerOrFirm = CustomerOrFirm.Firm, + ManualOrderIndicator = ManualOrderIndicator.Yes, + Account = account.name, + AccountId = account.id, + User = user.name, + UserId = user.id + }; + } } } \ No newline at end of file From 5f4eb58db925b56fdfa10c841b9b0a2810e5fbc6 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Fri, 8 Mar 2019 16:26:00 -0600 Subject: [PATCH 08/85] refactoring to promote eventual code reuse --- .../spark_test/System/IcebergAlgoTests.cs | 25 +++++++++------- src/client/spark_test/Utils/OrderUtilities.cs | 29 ++++++++----------- src/client/spark_test/Utils/TestUtils.cs | 24 ++++++++++++++- 3 files changed, 49 insertions(+), 29 deletions(-) diff --git a/src/client/spark_test/System/IcebergAlgoTests.cs b/src/client/spark_test/System/IcebergAlgoTests.cs index e91ca574f..4f7bf1008 100644 --- a/src/client/spark_test/System/IcebergAlgoTests.cs +++ b/src/client/spark_test/System/IcebergAlgoTests.cs @@ -34,7 +34,6 @@ namespace bts.tests { ServiceContext _context; AdminService _admin_service; OrderService _order_service; - Update _orders; Update _fills; public IcebergAlgoTests() { @@ -46,9 +45,6 @@ namespace bts.tests { _context, _admin_service, _order_service, - _order_service.subscribei_orders().Subscribe(ords => { - _orders = ords; - }), _order_service.subscribei_fills().Subscribe(phills => { // lol >8) @@ -146,22 +142,29 @@ namespace bts.tests { var ask_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Sell, test_account, test_user); orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 55.00, 50)); orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, 55.05, 50)); - await Task.Delay(new TimeSpan(0, 0, 3)); List test_orders = new List(); foreach (var i in orders_response_list) { - var iret = await OrderUtilities.get_filtered_orders(_orders, x => x.Key == i.value.OrderKey); - Assert.IsTrue(iret.Count() > 0 && iret.First().State.Status == OrderStatus.Working, "This order should have been 'working'"); + var iret = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.Key == i.value.OrderKey).Count() == 1, + rows => rows.Where(o => o.Key == i.value.OrderKey))).result ?? Enumerable.Empty(); + Assert.IsTrue(iret.Count() == 1 && iret.First().State.Status == OrderStatus.Working, "This order should have been 'working'"); test_orders.AddRange(iret); } // create an IB that will cross this market var ib_order_def = create_iceberg_order_def(s.Value.Id, Side.Buy, test_account, test_user); var ib_order_result = await OrderUtilities.send_order(_order_service, ib_order_def, 55.05, 30); - await Task.Delay(new TimeSpan(0, 0, 6)); - var ib_filtered_order = await OrderUtilities.get_filtered_orders(_orders, x => x.Key == ib_order_result.value.OrderKey); - var ib_child_orders = await OrderUtilities.get_filtered_orders(_orders, x => x.ParentKey == ib_order_result.value.OrderKey, 10); + var ib_filtered_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(x => x.Key == ib_order_result.value.OrderKey && x.HasState && x.State.Status == OrderStatus.Completed).Count() == 1, + rows => rows.Where(x => x.Key == ib_order_result.value.OrderKey))).result ?? Enumerable.Empty(); + var ib_child_orders = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(x => x.ParentKey == ib_order_result.value.OrderKey && x.HasState && x.State.Status == OrderStatus.Completed).Count() == 10, + rows => rows.Where(x => x.ParentKey == ib_order_result.value.OrderKey))).result ?? Enumerable.Empty(); - Assert.IsTrue(ib_filtered_order.Count() > 0 && ib_filtered_order.First().State.Status == OrderStatus.Completed, "This IB order should have been 'Completed'"); + Assert.IsTrue(ib_filtered_order.Count() == 1, "This IB order should have been 'Completed'"); Assert.IsTrue(ib_child_orders.Count() == 10, "There should have been 10 child orders"); Assert.IsTrue(ib_child_orders.All(x => x.HasState && x.State.Status == OrderStatus.Completed), "All child orders should be completed"); diff --git a/src/client/spark_test/Utils/OrderUtilities.cs b/src/client/spark_test/Utils/OrderUtilities.cs index fe099deba..fd1184938 100644 --- a/src/client/spark_test/Utils/OrderUtilities.cs +++ b/src/client/spark_test/Utils/OrderUtilities.cs @@ -13,6 +13,7 @@ using bts.api.orderman; using mbts; using mbts.tests; using bts.co2; +using bts.tests.utils; namespace bts.tests { @@ -44,24 +45,18 @@ namespace bts.tests } } - public static async Task> get_filtered_orders(Update orders, Func predicate, int count = 1) { - int current_count = 0; - int it = 0; - do { - if (orders != null) { - var ords = orders.all.Where(predicate); - current_count = ords.Count(); - - if (current_count >= count) { - return ords; - } - it++; - } - await Task.Delay(100); + public static Task>> + wait_for_filtered_orders(OrderService order_service, Func, bool> match, Func, IEnumerable> selector, TimeSpan? timeout = null) { + if (timeout == null) { + timeout = TimeSpan.FromSeconds(3); } - while (it < 100 && (orders == null || current_count < count)); - - return new List(); + var tcs = new TaskCompletionSource>(); + var s1 = order_service.subscribei_orders().Subscribe(update => { + if (match(update.all)) { + tcs.TrySetResult(selector(update.all)); + } + }) ; + return tcs.Task.timeout(timeout.Value); } } diff --git a/src/client/spark_test/Utils/TestUtils.cs b/src/client/spark_test/Utils/TestUtils.cs index f469031f3..71669a50f 100644 --- a/src/client/spark_test/Utils/TestUtils.cs +++ b/src/client/spark_test/Utils/TestUtils.cs @@ -2,9 +2,10 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; namespace bts.tests.utils { - static class TestUtils { + public static class TestUtils { public static List randomize_values(IReadOnlyList input) { var remaining_indices = Enumerable.Range(0, input.Count).ToList(); var rand = new Random(); @@ -38,5 +39,26 @@ namespace bts.tests.utils { } return reshuffled; } + + // this was copied from GlobalExtensions + public struct TimeoutResult { + public TimeoutResult(T result, TimeSpan ts) { + wait_duration = ts; + this.result = result; + } + public readonly TimeSpan wait_duration; + public readonly T result; + } + + // this was copied from GlobalExtensions + public static async Task> timeout(this Task task, TimeSpan timeout) { + System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch(); + sw.Start(); + var timeout_task = Task.Delay(timeout); + var completed = await Task.WhenAny(task, timeout_task); + return (completed == timeout_task) ? + new TimeoutResult(default(T), timeout) : + new TimeoutResult(task.Result, sw.Elapsed); + } } } \ No newline at end of file From 0e92a12e65fc6ddebebeab8895a87c11696a1bc8 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Mon, 11 Mar 2019 10:08:13 -0500 Subject: [PATCH 09/85] refactoring/reorg --- .../spark_test/System/IcebergAlgoTests.cs | 54 +++---------------- src/client/spark_test/Utils/OrderUtilities.cs | 50 +++++++++++++++-- src/client/spark_test/Utils/TestUtils.cs | 11 +++- 3 files changed, 64 insertions(+), 51 deletions(-) diff --git a/src/client/spark_test/System/IcebergAlgoTests.cs b/src/client/spark_test/System/IcebergAlgoTests.cs index 4f7bf1008..645a8700d 100644 --- a/src/client/spark_test/System/IcebergAlgoTests.cs +++ b/src/client/spark_test/System/IcebergAlgoTests.cs @@ -7,34 +7,20 @@ using System.Threading.Tasks; using bts.api; using bts.api.admin; using bts.api.core; -using bts.api.test; using bts.test; using bts.api.orderman; using mbts; -using mbts.tests; using bts.co2; +using bts.tests.utils; namespace bts.tests { - static class TestUtils { - public static bool test_price_equality(double price_1, double price_2, double tick_size) { - return Math.Abs(price_2 - price_1) < 0.01 * tick_size; - } - - // this is a copy from TickUtils -- ask what's legal for the spark_test project... - public static double ticksize(this ContractSpecificationImmutable spec, double price) => - Math.Abs(price) > spec.PriceBreak ? spec.HighTick : spec.LowTick; - - } - - [TestGroup(TestType = TestType.System)] class IcebergAlgoTests : IDisposable { List _resources; ServiceContext _context; AdminService _admin_service; OrderService _order_service; - Update _fills; public IcebergAlgoTests() { _context = new ServiceContext(); @@ -45,11 +31,6 @@ namespace bts.tests { _context, _admin_service, _order_service, - _order_service.subscribei_fills().Subscribe(phills => - { - // lol >8) - _fills = phills; - }) }; var auth = _context.login("bts", ""); @@ -94,28 +75,6 @@ namespace bts.tests { return security_with_no_market; } - public OrderDefinition create_iceberg_order_def(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user) { - return new OrderDefinition { - SecurityKey = security_id, - Side = side, - TimeInForce = TimeInForce.Day, - CustomDef = new CustomOrderDefinition { - Iceberg = new IcebergOrderDefinition { - ShowQuantity = 3, - Child = new CustomOrderDefinition { - Limit = new LimitOrderDefinition { } - } - } - }, - CtiCode = CTICode.CtiCode1, - CustomerOrFirm = CustomerOrFirm.Firm, - ManualOrderIndicator = ManualOrderIndicator.Yes, - Account = account.name, - AccountId = account.id, - User = user.name, - UserId = user.id - }; - } [Test(Timeout = 25)] public async Task test_ib_create_and_takeout_market() { @@ -153,7 +112,7 @@ namespace bts.tests { } // create an IB that will cross this market - var ib_order_def = create_iceberg_order_def(s.Value.Id, Side.Buy, test_account, test_user); + var ib_order_def = OrderDefinitionUtilities.create_iceberg_order_def(s.Value.Id, Side.Buy, test_account, test_user, 3); var ib_order_result = await OrderUtilities.send_order(_order_service, ib_order_def, 55.05, 30); var ib_filtered_order = (await OrderUtilities.wait_for_filtered_orders( _order_service, @@ -165,10 +124,13 @@ namespace bts.tests { rows => rows.Where(x => x.ParentKey == ib_order_result.value.OrderKey))).result ?? Enumerable.Empty(); Assert.IsTrue(ib_filtered_order.Count() == 1, "This IB order should have been 'Completed'"); - Assert.IsTrue(ib_child_orders.Count() == 10, "There should have been 10 child orders"); - Assert.IsTrue(ib_child_orders.All(x => x.HasState && x.State.Status == OrderStatus.Completed), "All child orders should be completed"); + Assert.IsTrue(ib_child_orders.Count() == 10, "There should have been 10 completed child orders"); - var filled_childen = _fills.all.Where(x => ib_child_orders.Select(o => o.Key).Contains(x.OrderKey)); + var filled_childen = (await OrderUtilities.wait_for_filtered_fills( + _order_service, + rows => rows.Where(x => ib_child_orders.Select(o => o.Key).Contains(x.OrderKey)).Count() == 10, + rows => rows.Where(x => ib_child_orders.Select(o => o.Key).Contains(x.OrderKey)))).result ?? Enumerable.Empty(); + Assert.IsTrue(filled_childen.Count() == 10, "There should have been 10 fills"); Assert.IsTrue(filled_childen.All(x => TestUtils.test_price_equality(x.LastPrice, 55.05, s.Value.ContractSpec.ticksize(x.LastPrice)) && x.LastShares == 3), "All child fills should have been filled at price 55.50 with quantity 3"); diff --git a/src/client/spark_test/Utils/OrderUtilities.cs b/src/client/spark_test/Utils/OrderUtilities.cs index fd1184938..c345554aa 100644 --- a/src/client/spark_test/Utils/OrderUtilities.cs +++ b/src/client/spark_test/Utils/OrderUtilities.cs @@ -45,18 +45,36 @@ namespace bts.tests } } - public static Task>> + public async static Task>> wait_for_filtered_orders(OrderService order_service, Func, bool> match, Func, IEnumerable> selector, TimeSpan? timeout = null) { if (timeout == null) { timeout = TimeSpan.FromSeconds(3); } var tcs = new TaskCompletionSource>(); - var s1 = order_service.subscribei_orders().Subscribe(update => { + var sub = order_service.subscribei_orders().Subscribe(update => { if (match(update.all)) { tcs.TrySetResult(selector(update.all)); } - }) ; - return tcs.Task.timeout(timeout.Value); + }); + var tr = await tcs.Task.timeout(timeout.Value); + sub.Dispose(); + return tr; + } + + public async static Task>> + wait_for_filtered_fills(OrderService order_service, Func, bool> match, Func, IEnumerable> selector, TimeSpan? timeout = null) { + if (timeout == null) { + timeout = TimeSpan.FromSeconds(3); + } + var tcs = new TaskCompletionSource>(); + var sub = order_service.subscribei_fills().Subscribe(update => { + if (match(update.all)) { + tcs.TrySetResult(selector(update.all)); + } + }); + var tr = await tcs.Task.timeout(timeout.Value); + sub.Dispose(); + return tr; } } @@ -80,5 +98,29 @@ namespace bts.tests UserId = user.id }; } + + public static OrderDefinition create_iceberg_order_def(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user, double show_quantity) { + return new OrderDefinition { + SecurityKey = security_id, + Side = side, + TimeInForce = TimeInForce.Day, + CustomDef = new CustomOrderDefinition { + Iceberg = new IcebergOrderDefinition { + ShowQuantity = show_quantity, + Child = new CustomOrderDefinition { + Limit = new LimitOrderDefinition { } + } + } + }, + CtiCode = CTICode.CtiCode1, + CustomerOrFirm = CustomerOrFirm.Firm, + ManualOrderIndicator = ManualOrderIndicator.Yes, + Account = account.name, + AccountId = account.id, + User = user.name, + UserId = user.id + }; + } } + } \ No newline at end of file diff --git a/src/client/spark_test/Utils/TestUtils.cs b/src/client/spark_test/Utils/TestUtils.cs index 71669a50f..dfdcdc5f4 100644 --- a/src/client/spark_test/Utils/TestUtils.cs +++ b/src/client/spark_test/Utils/TestUtils.cs @@ -1,4 +1,5 @@ -using bts.collections; +using bts.api; +using bts.collections; using System; using System.Collections.Generic; using System.Linq; @@ -40,6 +41,14 @@ namespace bts.tests.utils { return reshuffled; } + public static bool test_price_equality(double price_1, double price_2, double tick_size) { + return Math.Abs(price_2 - price_1) < 0.01 * tick_size; + } + + // this is a copy from TickUtils -- ask what's legal for the spark_test project... + public static double ticksize(this ContractSpecificationImmutable spec, double price) => + Math.Abs(price) > spec.PriceBreak ? spec.HighTick : spec.LowTick; + // this was copied from GlobalExtensions public struct TimeoutResult { public TimeoutResult(T result, TimeSpan ts) { From 316a617c65f39af2c0cc897e7d766c0e6c87a33c Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Mon, 11 Mar 2019 13:08:16 -0500 Subject: [PATCH 10/85] more refactoring --- .../spark_test/System/IcebergAlgoTests.cs | 45 +++---------------- .../spark_test/Utils/MarketDataUtilities.cs | 43 ++++++++++++++++++ 2 files changed, 48 insertions(+), 40 deletions(-) create mode 100644 src/client/spark_test/Utils/MarketDataUtilities.cs diff --git a/src/client/spark_test/System/IcebergAlgoTests.cs b/src/client/spark_test/System/IcebergAlgoTests.cs index 645a8700d..e6d9574bc 100644 --- a/src/client/spark_test/System/IcebergAlgoTests.cs +++ b/src/client/spark_test/System/IcebergAlgoTests.cs @@ -21,61 +21,26 @@ namespace bts.tests { ServiceContext _context; AdminService _admin_service; OrderService _order_service; + MarketDataRelay _md_relay; + public IcebergAlgoTests() { _context = new ServiceContext(); _admin_service = _context.locate(); _order_service = _context.locate(); + _md_relay = _context.locate(); _resources = new List { _context, _admin_service, _order_service, + _md_relay }; var auth = _context.login("bts", ""); auth.Wait(3000); } - - async Task get_security_with_no_market(string symbol) { - var request = new MDRequest() { - Host = "system_test", - Channels = { - new SecurityChannel() { - Time = 0, - Flags = (uint)MDRequest.Types.Flags.Bbo, - Type = SecurityType.Future, - Exchange = Venue.Sim, - Symbol = symbol, - TradeCursor = 0, - RfqCursor = 0 - } - } - }; - - var relay = _context.locate(); - await Task.Delay(100); - var result = await relay.requestmd_async(request); - - SecurityImmutable? security_with_no_market = new SecurityImmutable?(); - if (result.success) { - foreach (var ch in result.value.Channels) { - foreach (var update in ch.Updates) { - if (update.Direct.Asks.Count == 0 && update.Direct.Bids.Count == 0) { - security_with_no_market = new SecurityImmutable(update.Security); - break; - } - } - if (security_with_no_market.HasValue) { - break; - } - } - } - return security_with_no_market; - } - - [Test(Timeout = 25)] public async Task test_ib_create_and_takeout_market() { var test_user = (name: "bts", id: 0u); @@ -92,7 +57,7 @@ namespace bts.tests { test_user.id = user_data.Value.Id; // find a security with no market - var s = await get_security_with_no_market("CL"); + var s = await MarketDataUtilities.get_security(_md_relay, "CL", SecurityType.Future, update => update.Direct.Asks.Count == 0 && update.Direct.Bids.Count == 0); Assert.IsTrue(s.HasValue, "Could not find a security with no maket"); // create a market in the instrument diff --git a/src/client/spark_test/Utils/MarketDataUtilities.cs b/src/client/spark_test/Utils/MarketDataUtilities.cs new file mode 100644 index 000000000..b7b9c9781 --- /dev/null +++ b/src/client/spark_test/Utils/MarketDataUtilities.cs @@ -0,0 +1,43 @@ +using bts.api; +using bts.co2; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; + +namespace bts.tests { + class MarketDataUtilities { + + const string _MD_HOST = "system_test"; + + public static async Task get_security(MarketDataRelay relay, string symbol, SecurityType security_type, Func match, MDRequest.Types.Flags flags = MDRequest.Types.Flags.Bbo, Venue v = Venue.Sim) { + var request = new MDRequest() { + Host = _MD_HOST, + Channels = { + new SecurityChannel() { + Time = 0, + Flags = (uint)flags, + Type = security_type, + Exchange = v, + Symbol = symbol, + TradeCursor = 0, + RfqCursor = 0 + } + } + }; + var result = await relay.requestmd_async(request); + + if (result.success) { + foreach (var ch in result.value.Channels) { + foreach (var update in ch.Updates) { + if (match(update)) { + return new SecurityImmutable(update.Security); + } + } + } + } + return null; + } + + } +} From edfa079326e038c1a4ec9a9c2458f6f6bf1a5cdd Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Mon, 11 Mar 2019 13:46:00 -0500 Subject: [PATCH 11/85] ib avg filled test --- .../spark_test/System/IcebergAlgoTests.cs | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/client/spark_test/System/IcebergAlgoTests.cs b/src/client/spark_test/System/IcebergAlgoTests.cs index e6d9574bc..a16f5c4ae 100644 --- a/src/client/spark_test/System/IcebergAlgoTests.cs +++ b/src/client/spark_test/System/IcebergAlgoTests.cs @@ -104,6 +104,70 @@ namespace bts.tests { Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, test_orders), "Unable to cancel test orders"); } + [Test(Timeout = 25)] + public async Task test_ib_avg_fill() { + var test_user = (name: "bts", id: 0u); + var test_account = (name: "ACC A", id: 0u); + + // get the ACC A account + BTSAccountImmutable? account_data = await AdminUtilities.get_account_data(_admin_service, test_account.name); + Assert.IsTrue(account_data.HasValue); + test_account.id = account_data.Value.Id; + + // get the bts user + var user_data = await AdminUtilities.get_user_data(_admin_service, test_user.name); + Assert.IsTrue(user_data.HasValue); + test_user.id = user_data.Value.Id; + + // find a security with no market + var s = await MarketDataUtilities.get_security(_md_relay, "CL", SecurityType.Future, update => update.Direct.Asks.Count == 0 && update.Direct.Bids.Count == 0); + Assert.IsTrue(s.HasValue, "Could not find a security with no maket"); + + // create a market in the instrument + List> orders_response_list = new List>(); + var bid_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Buy, test_account, test_user); + var ask_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Sell, test_account, test_user); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, 56.02, 1)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, 56.01, 1)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, 56.00, 1)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, 55.50, 5)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, 55.01, 20)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 55.00, 1)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 54.99, 1)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 54.98, 1)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 54.97, 1)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 54.96, 1)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 54.95, 1)); + List test_orders = new List(); + foreach (var i in orders_response_list) { + var iret = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.Key == i.value.OrderKey).Count() == 1, + rows => rows.Where(o => o.Key == i.value.OrderKey))).result ?? Enumerable.Empty(); + Assert.IsTrue(iret.Count() == 1 && iret.First().State.Status == OrderStatus.Working, "This order should have been 'working'"); + test_orders.AddRange(iret); + } + + // create an IB that will cross this market + var ib_order_def = OrderDefinitionUtilities.create_iceberg_order_def(s.Value.Id, Side.Buy, test_account, test_user, 1); + var ib_order_result = await OrderUtilities.send_order(_order_service, ib_order_def, 56.02, 26); + var ib_filtered_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(x => x.Key == ib_order_result.value.OrderKey && x.HasState && x.State.Status == OrderStatus.Completed).Count() == 1, + rows => rows.Where(x => x.Key == ib_order_result.value.OrderKey))).result ?? Enumerable.Empty(); + var ib_child_orders = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(x => x.ParentKey == ib_order_result.value.OrderKey && x.HasState && x.State.Status == OrderStatus.Completed).Count() == 10, + rows => rows.Where(x => x.ParentKey == ib_order_result.value.OrderKey))).result ?? Enumerable.Empty(); + + Assert.IsTrue(ib_filtered_order.Count() == 1, "This IB order should have been 'Completed'"); + Assert.IsTrue(ib_filtered_order.All(x => + TestUtils.test_price_equality(x.State.AvgPrice, 55.1423, s.Value.ContractSpec.ticksize(x.State.AvgPrice))), "IB avg price should be 55.1423"); + + // cleanup + Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, test_orders), "Unable to cancel test orders"); + } + public void Dispose() { Disposable.dispose(_resources); } From 01a8502b20471a625d58b6a09867eb0066fe6606 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Mon, 11 Mar 2019 14:04:40 -0500 Subject: [PATCH 12/85] refactoring to resuse user and account between different tests --- .../spark_test/System/IcebergAlgoTests.cs | 53 ++++++++----------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/src/client/spark_test/System/IcebergAlgoTests.cs b/src/client/spark_test/System/IcebergAlgoTests.cs index a16f5c4ae..34f59345d 100644 --- a/src/client/spark_test/System/IcebergAlgoTests.cs +++ b/src/client/spark_test/System/IcebergAlgoTests.cs @@ -23,6 +23,8 @@ namespace bts.tests { OrderService _order_service; MarketDataRelay _md_relay; + (string name, uint id) _test_account = ("ACC A", 0u); + (string name, uint id) _test_user = ("bts", 0u); public IcebergAlgoTests() { _context = new ServiceContext(); @@ -39,31 +41,31 @@ namespace bts.tests { var auth = _context.login("bts", ""); auth.Wait(3000); + + // get the ACC A account + var account_data = AdminUtilities.get_account_data(_admin_service, _test_account.name); + account_data.Wait(3000); + Assert.IsTrue(account_data.Result.HasValue); + _test_account.id = account_data.Result.Value.Id; + + // get the bts user + var user_data = AdminUtilities.get_user_data(_admin_service, _test_user.name); + user_data.Wait(3000); + Assert.IsTrue(user_data.Result.HasValue); + _test_user.id = user_data.Result.Value.Id; + } [Test(Timeout = 25)] public async Task test_ib_create_and_takeout_market() { - var test_user = (name: "bts", id: 0u); - var test_account = (name: "ACC A", id: 0u); - - // get the ACC A account - BTSAccountImmutable? account_data = await AdminUtilities.get_account_data(_admin_service, test_account.name); - Assert.IsTrue(account_data.HasValue); - test_account.id = account_data.Value.Id; - - // get the bts user - var user_data = await AdminUtilities.get_user_data(_admin_service, test_user.name); - Assert.IsTrue(user_data.HasValue); - test_user.id = user_data.Value.Id; - // find a security with no market var s = await MarketDataUtilities.get_security(_md_relay, "CL", SecurityType.Future, update => update.Direct.Asks.Count == 0 && update.Direct.Bids.Count == 0); Assert.IsTrue(s.HasValue, "Could not find a security with no maket"); // create a market in the instrument List> orders_response_list = new List>(); - var bid_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Buy, test_account, test_user); - var ask_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Sell, test_account, test_user); + var bid_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Buy, _test_account, _test_user); + var ask_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Sell, _test_account, _test_user); orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 55.00, 50)); orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, 55.05, 50)); List test_orders = new List(); @@ -77,7 +79,7 @@ namespace bts.tests { } // create an IB that will cross this market - var ib_order_def = OrderDefinitionUtilities.create_iceberg_order_def(s.Value.Id, Side.Buy, test_account, test_user, 3); + var ib_order_def = OrderDefinitionUtilities.create_iceberg_order_def(s.Value.Id, Side.Buy, _test_account, _test_user, 3); var ib_order_result = await OrderUtilities.send_order(_order_service, ib_order_def, 55.05, 30); var ib_filtered_order = (await OrderUtilities.wait_for_filtered_orders( _order_service, @@ -106,27 +108,14 @@ namespace bts.tests { [Test(Timeout = 25)] public async Task test_ib_avg_fill() { - var test_user = (name: "bts", id: 0u); - var test_account = (name: "ACC A", id: 0u); - - // get the ACC A account - BTSAccountImmutable? account_data = await AdminUtilities.get_account_data(_admin_service, test_account.name); - Assert.IsTrue(account_data.HasValue); - test_account.id = account_data.Value.Id; - - // get the bts user - var user_data = await AdminUtilities.get_user_data(_admin_service, test_user.name); - Assert.IsTrue(user_data.HasValue); - test_user.id = user_data.Value.Id; - // find a security with no market var s = await MarketDataUtilities.get_security(_md_relay, "CL", SecurityType.Future, update => update.Direct.Asks.Count == 0 && update.Direct.Bids.Count == 0); Assert.IsTrue(s.HasValue, "Could not find a security with no maket"); // create a market in the instrument List> orders_response_list = new List>(); - var bid_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Buy, test_account, test_user); - var ask_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Sell, test_account, test_user); + var bid_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Buy, _test_account, _test_user); + var ask_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Sell, _test_account, _test_user); orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, 56.02, 1)); orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, 56.01, 1)); orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, 56.00, 1)); @@ -149,7 +138,7 @@ namespace bts.tests { } // create an IB that will cross this market - var ib_order_def = OrderDefinitionUtilities.create_iceberg_order_def(s.Value.Id, Side.Buy, test_account, test_user, 1); + var ib_order_def = OrderDefinitionUtilities.create_iceberg_order_def(s.Value.Id, Side.Buy, _test_account, _test_user, 1); var ib_order_result = await OrderUtilities.send_order(_order_service, ib_order_def, 56.02, 26); var ib_filtered_order = (await OrderUtilities.wait_for_filtered_orders( _order_service, From 84e443ee4b5d22c67f03fc45e7be53cb188da38a Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Wed, 13 Mar 2019 11:40:16 -0500 Subject: [PATCH 13/85] copying *.sh scripts for testing on linux --- tools/run_sparkcmd_test_Linux.cmd | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tools/run_sparkcmd_test_Linux.cmd b/tools/run_sparkcmd_test_Linux.cmd index 77b96532d..01186ea69 100644 --- a/tools/run_sparkcmd_test_Linux.cmd +++ b/tools/run_sparkcmd_test_Linux.cmd @@ -1,11 +1,18 @@ @echo off @setlocal SETLOCAL + +if [%1]==[] goto PRINT_USAGE + if [%3] == [r] ( set build_type=release ) else ( set build_type=debug ) +echo(%0) +echo(%1) +echo(%2) +echo(%3) echo %build_type% if "%SPARKVM_PORT%"=="" set SPARKVM_PORT=22 echo. @@ -16,6 +23,8 @@ dotnet publish -f netcoreapp2.1 cd ..\..\..\build\Debug\bin\netcoreapp2.1\publish bash -c "scp * bts@sparkvm:~/spark/build/ninja_%build_type%/server/" +copy /Y %~dp0\..\src\client\spark_test\system_test_scripts\*.sh ..\..\net471\ + cd ../../net471 bash -c "scp *.sh bts@sparkvm:~/spark/build/ninja_%build_type%/server/" @@ -23,3 +32,16 @@ ssh bts@sparkvm -p %SPARKVM_PORT% -o StrictHostKeyChecking=no -t "cd /home/bts/s ENDLOCAL popd +goto END + +:PRINT_USAGE +echo Runs a spark test on a linux machine. +echo( +echo usage: +echo run_sparkcmd_test_linux.cmd test_name [r] +echo( +echo r Specifies the build should be in release mode +echo( +goto END + +:END From e31b05042f48b9ec6a0fbc468c6c77165c4d6200 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Wed, 13 Mar 2019 16:46:58 -0500 Subject: [PATCH 14/85] refactoring common services into a base class; other misc refactoring --- .../spark_test/System/IcebergAlgoTests.cs | 53 ++---------------- src/client/spark_test/System/TestBase.cs | 56 +++++++++++++++++++ 2 files changed, 62 insertions(+), 47 deletions(-) create mode 100644 src/client/spark_test/System/TestBase.cs diff --git a/src/client/spark_test/System/IcebergAlgoTests.cs b/src/client/spark_test/System/IcebergAlgoTests.cs index 34f59345d..fc4e073aa 100644 --- a/src/client/spark_test/System/IcebergAlgoTests.cs +++ b/src/client/spark_test/System/IcebergAlgoTests.cs @@ -16,45 +16,8 @@ using bts.tests.utils; namespace bts.tests { [TestGroup(TestType = TestType.System)] - class IcebergAlgoTests : IDisposable { - List _resources; - ServiceContext _context; - AdminService _admin_service; - OrderService _order_service; - MarketDataRelay _md_relay; - - (string name, uint id) _test_account = ("ACC A", 0u); - (string name, uint id) _test_user = ("bts", 0u); - - public IcebergAlgoTests() { - _context = new ServiceContext(); - _admin_service = _context.locate(); - _order_service = _context.locate(); - _md_relay = _context.locate(); - - _resources = new List { - _context, - _admin_service, - _order_service, - _md_relay - }; - - var auth = _context.login("bts", ""); - auth.Wait(3000); - - // get the ACC A account - var account_data = AdminUtilities.get_account_data(_admin_service, _test_account.name); - account_data.Wait(3000); - Assert.IsTrue(account_data.Result.HasValue); - _test_account.id = account_data.Result.Value.Id; - - // get the bts user - var user_data = AdminUtilities.get_user_data(_admin_service, _test_user.name); - user_data.Wait(3000); - Assert.IsTrue(user_data.Result.HasValue); - _test_user.id = user_data.Result.Value.Id; - - } + class IcebergAlgoTests : TestBase { + public IcebergAlgoTests() { } [Test(Timeout = 25)] public async Task test_ib_create_and_takeout_market() { @@ -72,9 +35,9 @@ namespace bts.tests { foreach (var i in orders_response_list) { var iret = (await OrderUtilities.wait_for_filtered_orders( _order_service, - rows => rows.Where(o => o.Key == i.value.OrderKey).Count() == 1, + rows => rows.Where(o => o.Key == i.value.OrderKey && o.HasState && o.State.Status == OrderStatus.Working).Count() == 1, rows => rows.Where(o => o.Key == i.value.OrderKey))).result ?? Enumerable.Empty(); - Assert.IsTrue(iret.Count() == 1 && iret.First().State.Status == OrderStatus.Working, "This order should have been 'working'"); + Assert.IsTrue(iret.Count() == 1, "This order should have been 'working'"); test_orders.AddRange(iret); } @@ -131,9 +94,9 @@ namespace bts.tests { foreach (var i in orders_response_list) { var iret = (await OrderUtilities.wait_for_filtered_orders( _order_service, - rows => rows.Where(o => o.Key == i.value.OrderKey).Count() == 1, + rows => rows.Where(o => o.Key == i.value.OrderKey && o.HasState && o.State.Status == OrderStatus.Working).Count() == 1, rows => rows.Where(o => o.Key == i.value.OrderKey))).result ?? Enumerable.Empty(); - Assert.IsTrue(iret.Count() == 1 && iret.First().State.Status == OrderStatus.Working, "This order should have been 'working'"); + Assert.IsTrue(iret.Count() == 1, "This order should have been 'working'"); test_orders.AddRange(iret); } @@ -157,9 +120,5 @@ namespace bts.tests { Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, test_orders), "Unable to cancel test orders"); } - public void Dispose() { - Disposable.dispose(_resources); - } - } } diff --git a/src/client/spark_test/System/TestBase.cs b/src/client/spark_test/System/TestBase.cs new file mode 100644 index 000000000..fca6cbbed --- /dev/null +++ b/src/client/spark_test/System/TestBase.cs @@ -0,0 +1,56 @@ +using bts.api.admin; +using bts.api.core; +using bts.api.orderman; +using bts.co2; +using bts.test; +using mbts; +using System; +using System.Collections.Generic; +using System.Text; + +namespace bts.tests { + public abstract class TestBase : IDisposable { + + protected (string name, uint id) _test_account = ("ACC A", 0u); + protected (string name, uint id) _test_user = ("bts", 0u); + protected ServiceContext _context; + protected AdminService _admin_service; + protected OrderService _order_service; + protected MarketDataRelay _md_relay; + protected List _resources; + + public TestBase() { + _context = new ServiceContext(); + _admin_service = _context.locate(); + _order_service = _context.locate(); + _md_relay = _context.locate(); + + _resources = new List { + _context, + _admin_service, + _order_service, + _md_relay + }; + + var auth = _context.login("bts", ""); + auth.Wait(3000); + + // get the ACC A account + var account_data = AdminUtilities.get_account_data(_admin_service, _test_account.name); + account_data.Wait(3000); + Assert.IsTrue(account_data.Result.HasValue); + _test_account.id = account_data.Result.Value.Id; + + // get the bts user + var user_data = AdminUtilities.get_user_data(_admin_service, _test_user.name); + user_data.Wait(3000); + Assert.IsTrue(user_data.Result.HasValue); + _test_user.id = user_data.Result.Value.Id; + + } + + public void Dispose() { + Disposable.dispose(_resources); + } + } +} From ba1b667ae8e129a5701d15faf672f916e513304d Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Wed, 13 Mar 2019 16:47:23 -0500 Subject: [PATCH 15/85] basic join test --- src/client/spark_test/System/JoinAlgoTests.cs | 93 +++++++++++++++++++ src/client/spark_test/Utils/OrderUtilities.cs | 41 ++++++++ 2 files changed, 134 insertions(+) create mode 100644 src/client/spark_test/System/JoinAlgoTests.cs diff --git a/src/client/spark_test/System/JoinAlgoTests.cs b/src/client/spark_test/System/JoinAlgoTests.cs new file mode 100644 index 000000000..c348e9bb8 --- /dev/null +++ b/src/client/spark_test/System/JoinAlgoTests.cs @@ -0,0 +1,93 @@ +using bts.api; +using bts.api.core; +using bts.api.orderman; +using bts.test; +using bts.tests.utils; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace bts.tests { + + [TestGroup(TestType = TestType.System)] + class JoinAlgoTests : TestBase { + public JoinAlgoTests() { } + + [Test(Timeout = 25)] + public async Task test_join_create_and_takeout_market() { + double initial_price = 54.99; + double ask_price = 55.01; + + + // find a security with no market + var s = await MarketDataUtilities.get_security(_md_relay, "CL", SecurityType.Future, update => update.Direct.Asks.Count == 0 && update.Direct.Bids.Count == 0); + Assert.IsTrue(s.HasValue, "Could not find a security with no maket"); + + // create a market in the instrument + List> orders_response_list = new List>(); + var bid_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Buy, _test_account, _test_user); + var ask_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Sell, _test_account, _test_user); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 55.00, 10)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, ask_price, 1)); + List test_orders = new List(); + foreach (var i in orders_response_list) { + var iret = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.Key == i.value.OrderKey && o.HasState && o.State.Status == OrderStatus.Working).Count() == 1, + rows => rows.Where(o => o.Key == i.value.OrderKey))).result ?? Enumerable.Empty(); + Assert.IsTrue(iret.Count() == 1, "This order should have been 'working'"); + test_orders.AddRange(iret); + } + + // create a join order + var join_order_def = OrderDefinitionUtilities.create_join_order_definition(s.Value.Id, Side.Buy, _test_account, _test_user, new spark.QuantityMeasure() { Absolute = 1 }); + var join_order_response = await OrderUtilities.send_order(_order_service, join_order_def, initial_price, 1); + var join_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.Key == join_order_response.value.OrderKey && o.HasState && o.State.Status == OrderStatus.Working).Count() == 1, + rows => rows.Where(o => o.Key == join_order_response.value.OrderKey))).result ?? Enumerable.Empty(); + Assert.IsTrue(join_order.Count() == 1, "The join order should have 'working'"); + test_orders.Add(join_order.First()); + + // verify the join order's child is working at the specefied price + // note: the price doesn't actually matter in the join order + var join_child_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.ParentKey == join_order.First().Key && + o.HasState && (o.State.Status == OrderStatus.Working && + TestUtils.test_price_equality(o.State.Price, initial_price, s.Value.ContractSpec.ticksize(o.State.Price)))).Count() == 1, + rows => rows.Where(o => o.ParentKey == join_order.First().Key))).result ?? Enumerable.Empty(); + Assert.IsTrue(join_child_order.Count() == 1, $"The join child order should have been 'working' at {initial_price}"); + + // now we move the best bid to become the best offer + var original_bid_order = test_orders.Where(o => o.Def.Side == Side.Buy).First(); + var bid_order_response = await OrderUtilities.modify_order(_order_service, original_bid_order, ask_price); + var modified_bid_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.Key == bid_order_response.value.OrderKey && o.HasState && + TestUtils.test_price_equality(o.State.Price, ask_price, s.Value.ContractSpec.ticksize(o.State.Price))).Count() == 1, + rows => rows.Where(o => o.Key == bid_order_response.value.OrderKey))).result ?? Enumerable.Empty(); + Assert.IsTrue(modified_bid_order.Count() == 1, "The working bid should have become the best offer"); + + // and we finally verify that the child order has joined the market at the appropriate price + join_child_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.ParentKey == join_order.First().Key && + o.HasState && (o.State.Status == OrderStatus.Working && + TestUtils.test_price_equality(o.State.Price, ask_price, s.Value.ContractSpec.ticksize(o.State.Price)))).Count() == 1, + rows => rows.Where(o => o.ParentKey == join_order.First().Key))).result ?? Enumerable.Empty(); + Assert.IsTrue(join_child_order.Count() == 1, $"The join child order should now have been 'working' at {ask_price}"); + test_orders.Add(join_child_order.First()); + + // cleanup + Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, test_orders), "Unable to cancel test orders"); + + } + + + } + +} diff --git a/src/client/spark_test/Utils/OrderUtilities.cs b/src/client/spark_test/Utils/OrderUtilities.cs index c345554aa..6949c9c37 100644 --- a/src/client/spark_test/Utils/OrderUtilities.cs +++ b/src/client/spark_test/Utils/OrderUtilities.cs @@ -25,6 +25,22 @@ namespace bts.tests Quantity = size }); } + public static async Task> modify_order(OrderService order_service, OrderRecordImmutable o, uint size) { + return await modify_order(order_service, o.Key, o.State.Price, size); + } + public static async Task> modify_order(OrderService order_service, OrderRecordImmutable o, double price) { + return await modify_order(order_service, o.Key, price, o.State.Quantity); + } + public static async Task> modify_order(OrderService order_service, OrderRecordImmutable o, double price, uint size) { + return await modify_order(order_service, o.Key, price, size); + } + private static async Task> modify_order(OrderService order_service, uint order_key, double price, uint size) { + return await order_service.update_order_async(new Update() { + OrderKey = order_key, + Price = price, + Quantity = size + }); + } public static async Task cancel_orders(OrderService order_service, IEnumerable orders, uint key = 0){ if (key > 0) { @@ -121,6 +137,31 @@ namespace bts.tests UserId = user.id }; } + + public static OrderDefinition create_join_order_definition(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user, spark.QuantityMeasure join_threshold) { + return new OrderDefinition { + SecurityKey = security_id, + Side = side, + TimeInForce = TimeInForce.Day, + CustomDef = new CustomOrderDefinition { + Join = new JoinOrderDefinition { + JoinThreshold = join_threshold, + Child = new CustomOrderDefinition { + Limit = new LimitOrderDefinition { } + } + } + }, + CtiCode = CTICode.CtiCode1, + CustomerOrFirm = CustomerOrFirm.Firm, + ManualOrderIndicator = ManualOrderIndicator.Yes, + Account = account.name, + AccountId = account.id, + User = user.name, + UserId = user.id + }; + + } + } } \ No newline at end of file From 65b3d4fffa69a69def8cd44b799619953b45e97e Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Wed, 13 Mar 2019 16:58:00 -0500 Subject: [PATCH 16/85] refactoring OrderDefinition utils --- src/client/spark_test/Utils/OrderUtilities.cs | 62 +++++++------------ 1 file changed, 23 insertions(+), 39 deletions(-) diff --git a/src/client/spark_test/Utils/OrderUtilities.cs b/src/client/spark_test/Utils/OrderUtilities.cs index 6949c9c37..97566b503 100644 --- a/src/client/spark_test/Utils/OrderUtilities.cs +++ b/src/client/spark_test/Utils/OrderUtilities.cs @@ -99,12 +99,12 @@ namespace bts.tests /// We should replace with something like the OrderDataHeler from the Spark project. /// public static class OrderDefinitionUtilities { - public static OrderDefinition create_limit_order_def(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user) { + + private static OrderDefinition create_base_order(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user) { return new OrderDefinition { SecurityKey = security_id, Side = side, TimeInForce = TimeInForce.Day, - CustomDef = new CustomOrderDefinition { Limit = new LimitOrderDefinition { } }, CtiCode = CTICode.CtiCode1, CustomerOrFirm = CustomerOrFirm.Firm, ManualOrderIndicator = ManualOrderIndicator.Yes, @@ -114,52 +114,36 @@ namespace bts.tests UserId = user.id }; } + public static OrderDefinition create_limit_order_def(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user) { + var def = create_base_order(security_id, side, account, user); + def.CustomDef = new CustomOrderDefinition { Limit = new LimitOrderDefinition { } }; + return def; + } public static OrderDefinition create_iceberg_order_def(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user, double show_quantity) { - return new OrderDefinition { - SecurityKey = security_id, - Side = side, - TimeInForce = TimeInForce.Day, - CustomDef = new CustomOrderDefinition { - Iceberg = new IcebergOrderDefinition { - ShowQuantity = show_quantity, - Child = new CustomOrderDefinition { - Limit = new LimitOrderDefinition { } - } + var def = create_base_order(security_id, side, account, user); + def.CustomDef = new CustomOrderDefinition { + Iceberg = new IcebergOrderDefinition { + ShowQuantity = show_quantity, + Child = new CustomOrderDefinition { + Limit = new LimitOrderDefinition { } } - }, - CtiCode = CTICode.CtiCode1, - CustomerOrFirm = CustomerOrFirm.Firm, - ManualOrderIndicator = ManualOrderIndicator.Yes, - Account = account.name, - AccountId = account.id, - User = user.name, - UserId = user.id + } }; + return def; } public static OrderDefinition create_join_order_definition(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user, spark.QuantityMeasure join_threshold) { - return new OrderDefinition { - SecurityKey = security_id, - Side = side, - TimeInForce = TimeInForce.Day, - CustomDef = new CustomOrderDefinition { - Join = new JoinOrderDefinition { - JoinThreshold = join_threshold, - Child = new CustomOrderDefinition { - Limit = new LimitOrderDefinition { } - } + var def = create_base_order(security_id, side, account, user); + def.CustomDef = new CustomOrderDefinition { + Join = new JoinOrderDefinition { + JoinThreshold = join_threshold, + Child = new CustomOrderDefinition { + Limit = new LimitOrderDefinition { } } - }, - CtiCode = CTICode.CtiCode1, - CustomerOrFirm = CustomerOrFirm.Firm, - ManualOrderIndicator = ManualOrderIndicator.Yes, - Account = account.name, - AccountId = account.id, - User = user.name, - UserId = user.id + } }; - + return def; } } From b231a66fc76ac6f510cb4205ab8fee78be6eae9e Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Thu, 14 Mar 2019 11:06:59 -0500 Subject: [PATCH 17/85] need to make this test public --- src/client/spark_test/System/JoinAlgoTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark_test/System/JoinAlgoTests.cs b/src/client/spark_test/System/JoinAlgoTests.cs index c348e9bb8..71496d077 100644 --- a/src/client/spark_test/System/JoinAlgoTests.cs +++ b/src/client/spark_test/System/JoinAlgoTests.cs @@ -13,7 +13,7 @@ using System.Threading.Tasks; namespace bts.tests { [TestGroup(TestType = TestType.System)] - class JoinAlgoTests : TestBase { + public class JoinAlgoTests : TestBase { public JoinAlgoTests() { } [Test(Timeout = 25)] From 9744f53d65dcb9e1876a897b7cc6008095e227b8 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Thu, 14 Mar 2019 11:07:57 -0500 Subject: [PATCH 18/85] shell script that will perform setup work; will probably need to chmod this --- .../system_test_scripts/test_join_algo.sh | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 src/client/spark_test/system_test_scripts/test_join_algo.sh diff --git a/src/client/spark_test/system_test_scripts/test_join_algo.sh b/src/client/spark_test/system_test_scripts/test_join_algo.sh new file mode 100644 index 000000000..00fe07719 --- /dev/null +++ b/src/client/spark_test/system_test_scripts/test_join_algo.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash + +# Stop script on NZEC +set -e +# Stop script if unbound variable found (use ${var:-} if intentional) +set -u + +# Use in the the functions: eval $invocation +invocation='say_verbose "Calling: ${yellow:-}${FUNCNAME[0]} ${green:-}$*${normal:-}"' + +# standard output may be used as a return value in the functions +# we need a way to write text on the screen in the functions so that +# it won't interfere with the return value. +# Exposing stream 3 as a pipe to standard output of the script itself +exec 3>&1 + +# Setup some colors to use. These need to work in fairly limited shells, like the Ubuntu Docker container where there are only 8 colors. +# See if stdout is a terminal +if [ -t 1 ] && command -v tput > /dev/null; then + # see if it supports colors + ncolors=$(tput colors) + if [ -n "$ncolors" ] && [ $ncolors -ge 8 ]; then + bold="$(tput bold || echo)" + normal="$(tput sgr0 || echo)" + black="$(tput setaf 0 || echo)" + red="$(tput setaf 1 || echo)" + green="$(tput setaf 2 || echo)" + yellow="$(tput setaf 3 || echo)" + blue="$(tput setaf 4 || echo)" + magenta="$(tput setaf 5 || echo)" + cyan="$(tput setaf 6 || echo)" + white="$(tput setaf 7 || echo)" + fi +fi + +say_warning() { + printf "%b\n" "${yellow:-}$test_script_name: Warning: $1${normal:-}" +} + +say_err() { + printf "%b\n" "${red:-}$test_script_name: Error: $1${normal:-}" >&2 +} + +say() { + # using stream 3 (defined in the beginning) to not interfere with stdout of functions + # which may be used as return value + printf "%b\n" "${cyan:-}$test_script_name:${normal:-} $1" >&3 +} + +say_verbose() { + if [ "$verbose" = true ]; then + say "$1" + fi +} + +is_merge_with_md_on() { + eval $invocation + + if [ $(./sparkd configcheck ExchangeSim | grep merge_with_md: | cut -d " " -f 2) == "false" ]; then + return 0 + else + return 1 + fi +} + +set_merge_with_md_off() { + eval $invocation + + if [ -f $config_file_path ]; then + say_verbose "config file exists ... removing" + rm $config_file_path + fi + + echo "$merge_with_md_key" >> $config_file_path && test_config_file_created=true + + return 0 +} + +run_bts_test() { + eval $invocation + dotnet sparkcmd.dll test $test_script_name + return 0 +} + +test_script_name=test_join_create_and_takeout_market +verbose=true +config_file_path="/home/bts/sparkdata/conf/test_join_algo_tests.conf" +test_config_file_created=false +merge_with_md_key=$(/bin/cat <<'END_HEREDOC' +ExchangeSim { + value { + merge_with_md: false + } +} +END_HEREDOC +) + +if is_merge_with_md_on; then + set_merge_with_md_off + if is_merge_with_md_on; then + say_err "Could not turn off merge_with_md" + say_err "Please investigate manually" + exit 1 + fi +else + say_verbose "merge with md is already off" +fi + +run_bts_test + +if [ "$test_config_file_created" = true ]; then + rm $config_file_path +fi + +say "Test has finished." From 16e8a6e26df0b6bf2f0339ba3ec6a2d127541d80 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Thu, 14 Mar 2019 15:50:42 -0500 Subject: [PATCH 19/85] adding .NET Core Remote Lauch and Attach debugging options --- .vscode/launch.json | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.vscode/launch.json b/.vscode/launch.json index 0cd375b54..d1fd262ff 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -275,5 +275,42 @@ "RedirectOutput" ] }, + { + "name": ".NET Core Remote Attach", + "type": "coreclr", + "request": "attach", + "processId": "${command:pickRemoteProcess}", + "pipeTransport": { + "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe", + "pipeArgs": ["bts@sparkvm"], + "debuggerPath": "/home/bts/vsdbg/vsdbg", + "pipeCwd": "${workspaceRoot}", + "quoteArgs": true + }, + "sourceFileMap": { + "/home/bts/spark/": "${workspaceRoot}" + }, + "justMyCode": true + }, + { + "name": ".NET Core Remote Launch", + "type": "coreclr", + "request": "launch", + "program": "/usr/bin/dotnet", + "args": ["sparkcmd.dll", "test", "test_join_create_and_takeout_market"], + "stopAtEntry": false, + "cwd": "/home/bts/spark/build/ninja_debug/server", + "pipeTransport": { + "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe", + "pipeArgs": ["bts@sparkvm"], + "debuggerPath": "/home/bts/vsdbg/vsdbg", + "pipeCwd": "${workspaceRoot}", + "quoteArgs": true + }, + "sourceFileMap": { + "/home/bts/spark/": "${workspaceRoot}" + }, + "justMyCode": false + }, ] } \ No newline at end of file From f6eb1e47459df40251e47fcd89c6bb7a9089339b Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Thu, 14 Mar 2019 15:53:12 -0500 Subject: [PATCH 20/85] QuickBusClientChanel diag logging for sub / unsub --- .../spark_api/api/Channels/QuickBusClientChannel.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/client/spark_api/api/Channels/QuickBusClientChannel.cs b/src/client/spark_api/api/Channels/QuickBusClientChannel.cs index ac054c2c8..a6d96b726 100644 --- a/src/client/spark_api/api/Channels/QuickBusClientChannel.cs +++ b/src/client/spark_api/api/Channels/QuickBusClientChannel.cs @@ -67,8 +67,8 @@ namespace bts.api.core { while (true) { var response = await check_stream_access(method_id, qualifier); if (response.success) { - if (response.value.Authorized) { - // success. Break loop and subscribe + if (response.value.Authorized) { + // success. Break loop and subscribe break; } else { @@ -94,7 +94,8 @@ namespace bts.api.core { // Authorized. Stream events unsubscribe_temp_sub(); return SubscribeNative(method, qualifier, parser, converter, observer); - }); + }); + Log.QBus.diag($"Subscribing for sub: '{method}/{qualifier}'"); return result; } private Task> check_stream_access(int method_id, string qualifier) { @@ -143,6 +144,7 @@ namespace bts.api.core { }; var subscription = ApiNativeMethods.api_client_subscribe(_client, method, qualifier, cb, IntPtr.Zero); return FinalizeableAnonymousDisposable.Create((disposing) => { + Log.QBus.diag($"Unsubscribing for sub: '{method}/{qualifier}'"); ApiNativeMethods.api_client_unsubscribe(_client, subscription); subscription = IntPtr.Zero; GC.KeepAlive(cb); From cb8dac05d5c48c6957c9dfce39fcd4c573958ccf Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Thu, 14 Mar 2019 16:10:58 -0500 Subject: [PATCH 21/85] if linux no Dispose for now b/c seg fault --- src/client/spark_test/System/TestBase.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/client/spark_test/System/TestBase.cs b/src/client/spark_test/System/TestBase.cs index fca6cbbed..dad725590 100644 --- a/src/client/spark_test/System/TestBase.cs +++ b/src/client/spark_test/System/TestBase.cs @@ -50,7 +50,10 @@ namespace bts.tests { } public void Dispose() { - Disposable.dispose(_resources); + // seg faults on linux during this dispose, not sure why yet + if (System.Environment.OSVersion.Platform == PlatformID.Win32NT) { + Disposable.dispose(_resources); + } } } } From f2f8c258f9eeade4ca21f9e02ab1c07ed21735ae Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Fri, 15 Mar 2019 13:22:37 -0500 Subject: [PATCH 22/85] run all tests within a test class --- .../system_test_scripts/test_join_algo.sh | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/src/client/spark_test/system_test_scripts/test_join_algo.sh b/src/client/spark_test/system_test_scripts/test_join_algo.sh index 00fe07719..a9dd79390 100644 --- a/src/client/spark_test/system_test_scripts/test_join_algo.sh +++ b/src/client/spark_test/system_test_scripts/test_join_algo.sh @@ -76,13 +76,38 @@ set_merge_with_md_off() { return 0 } -run_bts_test() { +# args +# return_var - $1 +# class_name - $2 +get_tests_in_class() { eval $invocation - dotnet sparkcmd.dll test $test_script_name + + local return_var=$1 + local class_name=$2 + local arr=($(dotnet sparkcmd.dll test --system --list | perl -ne '/^( *)/; $i = length $1; $j && $i >= $j and print, next; $j = 0; /'"$class_name"'/ and $j = $i + 1, print' | tail -n +2)) + eval $return_var='("${arr[@]}")' + return 0 } -test_script_name=test_join_create_and_takeout_market +# args +# class_name - $1 +run_bts_test() { + eval $invocation + + class_name=$1 + + local test_names_tuple=() + get_tests_in_class test_names_tuple $class_name + for test_name in ${test_names_tuple[@]}; do + say_verbose "running test with name: $test_name ..."; + dotnet sparkcmd.dll test $test_name + done + + return 0 +} + +test_script_name=$(basename "$0") verbose=true config_file_path="/home/bts/sparkdata/conf/test_join_algo_tests.conf" test_config_file_created=false @@ -106,7 +131,7 @@ else say_verbose "merge with md is already off" fi -run_bts_test +run_bts_test "JoinAlgoTests" if [ "$test_config_file_created" = true ]; then rm $config_file_path From 5cb48fb517cedfd3f732efff761ff2ef9a322b3f Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Fri, 15 Mar 2019 13:28:30 -0500 Subject: [PATCH 23/85] Adding qty threshold test --- src/client/spark_test/System/JoinAlgoTests.cs | 91 ++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/src/client/spark_test/System/JoinAlgoTests.cs b/src/client/spark_test/System/JoinAlgoTests.cs index 71496d077..859904888 100644 --- a/src/client/spark_test/System/JoinAlgoTests.cs +++ b/src/client/spark_test/System/JoinAlgoTests.cs @@ -17,7 +17,7 @@ namespace bts.tests { public JoinAlgoTests() { } [Test(Timeout = 25)] - public async Task test_join_create_and_takeout_market() { + public async Task test_join_price() { double initial_price = 54.99; double ask_price = 55.01; @@ -87,6 +87,95 @@ namespace bts.tests { } + [Test(Timeout = 25)] + public async Task test_join_qty_threshold() { + double initial_price = 54.99; + double ask_price = 55.01; + uint qty_threshold = 15; + + + // find a security with no market + var s = await MarketDataUtilities.get_security(_md_relay, "CL", SecurityType.Future, update => update.Direct.Asks.Count == 0 && update.Direct.Bids.Count == 0); + Assert.IsTrue(s.HasValue, "Could not find a security with no maket"); + + // create a market in the instrument + List> orders_response_list = new List>(); + var bid_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Buy, _test_account, _test_user); + var ask_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Sell, _test_account, _test_user); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 55.00, 10)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, ask_price, 1)); + List test_orders = new List(); + foreach (var i in orders_response_list) { + var iret = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.Key == i.value.OrderKey && o.HasState && o.State.Status == OrderStatus.Working).Count() == 1, + rows => rows.Where(o => o.Key == i.value.OrderKey))).result ?? Enumerable.Empty(); + Assert.IsTrue(iret.Count() == 1, "This order should have been 'working'"); + test_orders.AddRange(iret); + } + + // create a join order + var join_order_def = OrderDefinitionUtilities.create_join_order_definition(s.Value.Id, Side.Buy, _test_account, _test_user, new spark.QuantityMeasure() { Absolute = qty_threshold }); + var join_order_response = await OrderUtilities.send_order(_order_service, join_order_def, initial_price, 1); + var join_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.Key == join_order_response.value.OrderKey && o.HasState && o.State.Status == OrderStatus.Working).Count() == 1, + rows => rows.Where(o => o.Key == join_order_response.value.OrderKey))).result ?? Enumerable.Empty(); + Assert.IsTrue(join_order.Count() == 1, "The join order should have 'working'"); + test_orders.Add(join_order.First()); + + // verify the join order's child is working at the specefied price + // note: the price doesn't actually matter in the join order + var join_child_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.ParentKey == join_order.First().Key && + o.HasState && (o.State.Status == OrderStatus.Working && + TestUtils.test_price_equality(o.State.Price, initial_price, s.Value.ContractSpec.ticksize(o.State.Price)))).Count() == 1, + rows => rows.Where(o => o.ParentKey == join_order.First().Key))).result ?? Enumerable.Empty(); + Assert.IsTrue(join_child_order.Count() == 1, $"The join child order should have been 'working' at {initial_price}"); + + // now we move the best bid to become the best offer + var top_bid_order = test_orders.Where(o => o.Def.Side == Side.Buy).First(); + var bid_order_response = await OrderUtilities.modify_order(_order_service, top_bid_order, ask_price); + var modified_bid_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.Key == bid_order_response.value.OrderKey && o.HasState && + TestUtils.test_price_equality(o.State.Price, ask_price, s.Value.ContractSpec.ticksize(o.State.Price))).Count() == 1, + rows => rows.Where(o => o.Key == bid_order_response.value.OrderKey))).result ?? Enumerable.Empty(); + Assert.IsTrue(modified_bid_order.Count() == 1, "The working bid should have become the best offer"); + + // verify the join order's child is *still* working at the initial price + // b/c the quantity threshold has not yet been met + join_child_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.ParentKey == join_order.First().Key && + o.HasState && (o.State.Status == OrderStatus.Working && + TestUtils.test_price_equality(o.State.Price, initial_price, s.Value.ContractSpec.ticksize(o.State.Price)))).Count() == 1, + rows => rows.Where(o => o.ParentKey == join_order.First().Key))).result ?? Enumerable.Empty(); + Assert.IsTrue(join_child_order.Count() == 1, $"The join child order should still have been 'working' at {initial_price}"); + + // now we modify our top bid's volume to meet the join's threshold + bid_order_response = await OrderUtilities.modify_order(_order_service, modified_bid_order.First(), qty_threshold); + modified_bid_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.Key == bid_order_response.value.OrderKey && o.HasState && o.State.Quantity == qty_threshold).Count() == 1, + rows => rows.Where(o => o.Key == bid_order_response.value.OrderKey))).result ?? Enumerable.Empty(); + Assert.IsTrue(modified_bid_order.Count() == 1, $"The working bid should have its quantity modified to {qty_threshold}"); + + // now that the qty threshold has been met, verify that the child order has joined the market at the appropriate price + join_child_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.ParentKey == join_order.First().Key && + o.HasState && (o.State.Status == OrderStatus.Working && + TestUtils.test_price_equality(o.State.Price, ask_price, s.Value.ContractSpec.ticksize(o.State.Price)))).Count() == 1, + rows => rows.Where(o => o.ParentKey == join_order.First().Key))).result ?? Enumerable.Empty(); + Assert.IsTrue(join_child_order.Count() == 1, $"The join child order should now have been 'working' at {ask_price}"); + test_orders.Add(join_child_order.First()); + + // cleanup + Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, test_orders), "Unable to cancel test orders"); + } + } From db1b2a63af4fa008e78e0474db2451e0d4b48ce5 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Thu, 21 Mar 2019 08:05:33 -0500 Subject: [PATCH 24/85] the start of a lean test --- src/client/spark_test/System/LeanAlgoTests.cs | 144 ++++++++++++++++++ src/client/spark_test/Utils/OrderUtilities.cs | 15 ++ 2 files changed, 159 insertions(+) create mode 100644 src/client/spark_test/System/LeanAlgoTests.cs diff --git a/src/client/spark_test/System/LeanAlgoTests.cs b/src/client/spark_test/System/LeanAlgoTests.cs new file mode 100644 index 000000000..2f39c7331 --- /dev/null +++ b/src/client/spark_test/System/LeanAlgoTests.cs @@ -0,0 +1,144 @@ +using bts.api; +using bts.api.core; +using bts.api.orderman; +using bts.test; +using bts.tests.utils; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace bts.tests { + + [TestGroup(TestType = TestType.System)] + public class LeanAlgoTests : TestBase { + + public LeanAlgoTests() { } + + [Test(Timeout = 25)] + public async Task test_lean_price() { + double initial_price = 54.98; + double ask_price_1 = 55.00; + double ask_price_2 = 55.01; + uint ask_size = 10; + uint lean_order_size = 30; + uint cross_threshold = ask_size - 1; + + // find a security with no market + var s = await MarketDataUtilities.get_security(_md_relay, "CL", SecurityType.Future, update => update.Direct.Asks.Count == 0 && update.Direct.Bids.Count == 0); + Assert.IsTrue(s.HasValue, "Could not find a security with no maket"); + + // create a market in the instrument + List> orders_response_list = new List>(); + var bid_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Buy, _test_account, _test_user); + var ask_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Sell, _test_account, _test_user); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 54.99, 1)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, ask_price_1, ask_size)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, ask_price_2, ask_size)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, ask_price_2 + 0.01, ask_size)); + orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, ask_price_2 + 0.02, ask_size)); + List test_orders = new List(); + foreach (var i in orders_response_list) { + var iret = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.Key == i.value.OrderKey && o.HasState && o.State.Status == OrderStatus.Working).Count() == 1, + rows => rows.Where(o => o.Key == i.value.OrderKey))).result ?? Enumerable.Empty(); + Assert.IsTrue(iret.Count() == 1, "This order should have been 'working'"); + test_orders.AddRange(iret); + } + + // create a lean order, 2 tick chase (default) + var lean_order_def = OrderDefinitionUtilities.create_lean_order_definition(s.Value.Id, Side.Buy, _test_account, _test_user, new spark.QuantityMeasure() { Absolute = cross_threshold }); + var lean_order_response = await OrderUtilities.send_order(_order_service, lean_order_def, initial_price, lean_order_size); + var lean_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.Key == lean_order_response.value.OrderKey && o.HasState && o.State.Status == OrderStatus.Working).Count() == 1, + rows => rows.Where(o => o.Key == lean_order_response.value.OrderKey))).result ?? Enumerable.Empty(); + Assert.IsTrue(lean_order.Count() == 1, "The lean order should have 'working'"); + test_orders.Add(lean_order.First()); + + // verify the lean order's child is working at the specified price + // note: the price doesn't actually matter in the lean order + var lean_child_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.ParentKey == lean_order.First().Key && + o.HasState && (o.State.Status == OrderStatus.Working && + TestUtils.test_price_equality(o.State.Price, initial_price, s.Value.ContractSpec.ticksize(o.State.Price)))).Count() == 1, + rows => rows.Where(o => o.ParentKey == lean_order.First().Key))).result ?? Enumerable.Empty(); + Assert.IsTrue(lean_child_order.Count() == 1, $"The lean child order should have been 'working' at {initial_price}"); + + // now we modify the opposite inside market qty to breach the lean threshold + var target_ask_order = test_orders.Where(o => o.Def.Side == Side.Sell && + TestUtils.test_price_equality(o.State.Price, ask_price_1, s.Value.ContractSpec.ticksize(o.State.Price))).First(); + var ask_order_response = await OrderUtilities.modify_order(_order_service, target_ask_order, cross_threshold); + var modified_ask_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.Key == ask_order_response.value.OrderKey && o.HasState && + o.State.Quantity == cross_threshold).Count() == 1, + rows => rows.Where(o => o.Key == ask_order_response.value.OrderKey))).result ?? Enumerable.Empty(); + Assert.IsTrue(modified_ask_order.Count() == 1, "The opposite inside market should have been adjusted to the Lean's cross threshold"); + + // verify that the child order has leaned into the market at the appropriate price and been parially filled appropriately + // this is the initial trigger payup + lean_child_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.ParentKey == lean_order.First().Key && + o.HasState && ( + o.State.Status == OrderStatus.Working && + TestUtils.test_price_equality(o.State.Price, ask_price_1, s.Value.ContractSpec.ticksize(o.State.Price)) && + o.State.FilledQuantity == 9) + ).Count() == 1, + rows => rows.Where(o => o.ParentKey == lean_order.First().Key))).result ?? Enumerable.Empty(); + Assert.IsTrue(lean_child_order.Count() == 1, $"Initial trigger: The lean child order should now have been 'working' at {ask_price_1}"); + test_orders.Add(lean_child_order.First()); + + // now we modify the opposite inside market qty again to start checking that the case is working + target_ask_order = test_orders.Where(o => o.Def.Side == Side.Sell && + TestUtils.test_price_equality(o.State.Price, ask_price_2, s.Value.ContractSpec.ticksize(o.State.Price))).First(); + ask_order_response = await OrderUtilities.modify_order(_order_service, target_ask_order, cross_threshold); + modified_ask_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.Key == ask_order_response.value.OrderKey && o.HasState && + o.State.Quantity == cross_threshold).Count() == 1, + rows => rows.Where(o => o.Key == ask_order_response.value.OrderKey))).result ?? Enumerable.Empty(); + Assert.IsTrue(modified_ask_order.Count() == 1, "The opposite inside market should have been adjusted to the Lean's cross threshold"); + + // verify that the child order has chased and that it's filled quantity has increased + lean_child_order = (await OrderUtilities.wait_for_filtered_orders( + _order_service, + rows => rows.Where(o => o.ParentKey == lean_order.First().Key && + o.HasState && ( + o.State.Status == OrderStatus.Working && + TestUtils.test_price_equality(o.State.Price, ask_price_2, s.Value.ContractSpec.ticksize(o.State.Price)) && + o.State.FilledQuantity == 9) + ).Count() == 1, + rows => rows.Where(o => o.ParentKey == lean_order.First().Key))).result ?? Enumerable.Empty(); + Assert.IsTrue(lean_child_order.Count() == 1, $"First Chase: The lean child order should now have been 'working' at {ask_price_1}"); + test_orders.Add(lean_child_order.First()); + + // TODO: + // verify the parent order also reflects the increase filled quantity from the child order + + // TODO: + // instigate another chase + + // TODO: + // verify parent order again reflects + + // TODO: + // instigate another chanse + + // TODO: + // verify we are *not* chasing any longer, verify appropriate order statuses + + + + // cleanup + Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, test_orders), "Unable to cancel test orders"); + + } + + } +} diff --git a/src/client/spark_test/Utils/OrderUtilities.cs b/src/client/spark_test/Utils/OrderUtilities.cs index 97566b503..b64bc5662 100644 --- a/src/client/spark_test/Utils/OrderUtilities.cs +++ b/src/client/spark_test/Utils/OrderUtilities.cs @@ -146,6 +146,21 @@ namespace bts.tests return def; } + public static OrderDefinition create_lean_order_definition(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user, spark.QuantityMeasure lean_threshold, uint max_chase_ticks = 2, LeanOrderDefinition.Types.OnToward ontoward_behavor = LeanOrderDefinition.Types.OnToward.NoChange) { + var def = create_base_order(security_id, side, account, user); + def.CustomDef = new CustomOrderDefinition { + Lean = new LeanOrderDefinition { + CrossThreshold = lean_threshold, + MaxChaseTicks = max_chase_ticks, + TradeTowardBehavior = ontoward_behavor, + Child = new CustomOrderDefinition { + Limit = new LimitOrderDefinition { } + } + } + }; + return def; + } + } } \ No newline at end of file From 4b8d725528866e8b8ca43e3da98cae3397dd0d1b Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Fri, 22 Mar 2019 10:58:40 -0400 Subject: [PATCH 25/85] Test scripts start sparkd. spreader sets config --- .../system_test_scripts/spreader_tests.sh | 32 +++++++++++++++++++ .../system_test_scripts/test_clip_limits.sh | 14 ++++++++ .../test_position_limits.sh | 10 ++++++ 3 files changed, 56 insertions(+) create mode 100644 src/client/spark_test/system_test_scripts/spreader_tests.sh create mode 100644 src/client/spark_test/system_test_scripts/test_clip_limits.sh diff --git a/src/client/spark_test/system_test_scripts/spreader_tests.sh b/src/client/spark_test/system_test_scripts/spreader_tests.sh new file mode 100644 index 000000000..10fad4d10 --- /dev/null +++ b/src/client/spark_test/system_test_scripts/spreader_tests.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +rm -r ~/sparkdata/data/databases/ > /dev/null + +mv ~/sparkdata/conf/spark.conf ~/sparkdata/conf/spark.conf.bak +cat << EOF > ~/sparkdata/conf/spark.conf + ExchangeSim { + value { + merge_with_md:false + } + } + Logging { + value { + console { default_level: "Error" } + syslog { default_level: "Error" } + file { default_level: "Error" } + } +} +EOF + + +nohup ./sparkd run & +pid=$! +echo "sparkd pid: " $pid + +echo "sleeping for 120 seconds..." +sleep 120 + +dotnet sparkcmd.dll test spreader_test + +kill $pid +mv ~/sparkdata/conf/spark.conf.bak ~/sparkdata/conf/spark.conf \ No newline at end of file diff --git a/src/client/spark_test/system_test_scripts/test_clip_limits.sh b/src/client/spark_test/system_test_scripts/test_clip_limits.sh new file mode 100644 index 000000000..e90607b04 --- /dev/null +++ b/src/client/spark_test/system_test_scripts/test_clip_limits.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +rm -r ~/sparkdata/data/databases/ > /dev/null + +nohup ./sparkd run & +pid=$! +echo "sparkd pid: " $pid + +echo "sleeping for 120 seconds..." +sleep 120 + +dotnet sparkcmd.dll test test_clip_limits + +kill $pid \ No newline at end of file diff --git a/src/client/spark_test/system_test_scripts/test_position_limits.sh b/src/client/spark_test/system_test_scripts/test_position_limits.sh index 899de5fd9..06efd0c0c 100644 --- a/src/client/spark_test/system_test_scripts/test_position_limits.sh +++ b/src/client/spark_test/system_test_scripts/test_position_limits.sh @@ -1,3 +1,13 @@ #!/bin/bash +rm -r ~/sparkdata/data/databases/ > /dev/null + +nohup ./sparkd run & +pid=$! +echo "sparkd pid: " $pid + +echo "sleeping for 120 seconds..." +sleep 120 dotnet sparkcmd.dll test test_position_limits + +kill $pid \ No newline at end of file From dd7db5661bf67a48347c6571785b814396aec681 Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Fri, 22 Mar 2019 11:22:56 -0400 Subject: [PATCH 26/85] Utility for tests to verify the system is ready --- src/client/spark_test/Utils/TestUtils.cs | 85 +++++++++++++++++++++--- 1 file changed, 77 insertions(+), 8 deletions(-) diff --git a/src/client/spark_test/Utils/TestUtils.cs b/src/client/spark_test/Utils/TestUtils.cs index dfdcdc5f4..6967bee52 100644 --- a/src/client/spark_test/Utils/TestUtils.cs +++ b/src/client/spark_test/Utils/TestUtils.cs @@ -1,8 +1,12 @@ using bts.api; +using bts.api.admin; +using bts.api.core; +using bts.api.orderman; using bts.collections; using System; using System.Collections.Generic; using System.Linq; +using System.Reactive.Linq; using System.Threading.Tasks; namespace bts.tests.utils { @@ -60,14 +64,79 @@ namespace bts.tests.utils { } // this was copied from GlobalExtensions - public static async Task> timeout(this Task task, TimeSpan timeout) { - System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch(); - sw.Start(); - var timeout_task = Task.Delay(timeout); - var completed = await Task.WhenAny(task, timeout_task); - return (completed == timeout_task) ? - new TimeoutResult(default(T), timeout) : - new TimeoutResult(task.Result, sw.Elapsed); + public static async Task> timeout(this Task task, TimeSpan timeout) { + System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch(); + sw.Start(); + var timeout_task = Task.Delay(timeout); + var completed = await Task.WhenAny(task, timeout_task); + return (completed == timeout_task) ? + new TimeoutResult(default(T), timeout) : + new TimeoutResult(task.Result, sw.Elapsed); + } + + public static async Task await_with_timeout(Task task, TimeSpan timeout){ + var to = timeout(task, timeout); + var tr = await to; + return tr.result; + } + + + public static async Task test_for_system_ready(ServiceContext context, TimeSpan? timeout = null) { + if(timeout == null) { + timeout = TimeSpan.FromSeconds(30); + } + + System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch(); + sw.Start(); + + bool admin_ready = await test_for_admin_service_ready(context, timeout.Value.Subtract(sw.Elapsed)); + Console.WriteLine("Admin Service Ready: " + admin_ready); + bool md_ready = await test_for_md_service_ready(context, timeout.Value.Subtract(sw.Elapsed)); + Console.WriteLine("MD Service Ready: " + md_ready); + + return admin_ready && md_ready; + } + + public static async Task test_for_admin_service_ready(ServiceContext context, TimeSpan timeout) { + System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch(); + sw.Start(); + + var admin_service = context.locate(); + + Update admin_value = null; + var account_sub = admin_service.subscribei_admin_data().Subscribe(update => admin_value = update); + + while(admin_value == null && sw.Elapsed < timeout) { + Console.WriteLine("Waiting for admin service..."); + await Task.Delay(100); + } + + return admin_value != null; + } + + + public static async Task test_for_md_service_ready(ServiceContext context, TimeSpan timeout) { + var md_request = new co2.MDRequest() { + Host = "system_test", + Channels = { + new co2.SecurityChannel() { + Time = 0, + Flags = 0, + Type = SecurityType.Future, + Exchange = Venue.Sim, + Symbol = "*", + TradeCursor = 0, + RfqCursor = 0 + } + } + }; + + var md_relay = context.locate(); + Result? md_result = await await_with_timeout( + md_relay.requestmd_async(md_request), + timeout); + + return md_result != null; } } } \ No newline at end of file From c1fe0b319100e64f12a4f5c6348c38019663c69d Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Thu, 4 Apr 2019 11:56:05 -0400 Subject: [PATCH 27/85] SPARK-2066: Create book and common setup --- src/client/spark_test/System/SpreaderTests.cs | 67 ++++++++++++++ .../spark_test/Utils/MarketDataUtilities.cs | 31 +++++++ src/client/spark_test/Utils/OrderUtilities.cs | 29 +++++- .../spark_test/Utils/SpreaderUtilities.cs | 89 +++++++++++++++++++ 4 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 src/client/spark_test/System/SpreaderTests.cs create mode 100644 src/client/spark_test/Utils/SpreaderUtilities.cs diff --git a/src/client/spark_test/System/SpreaderTests.cs b/src/client/spark_test/System/SpreaderTests.cs new file mode 100644 index 000000000..03d0cde7f --- /dev/null +++ b/src/client/spark_test/System/SpreaderTests.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive.Linq; +using System.Text; +using System.Threading.Tasks; +using bts.api; +using bts.api.admin; +using bts.api.core; +using bts.api.test; +using bts.test; +using bts.api.orderman; +using mbts; +using mbts.tests; +using bts.co2; +using bts.tests.utils; + +namespace bts.tests +{ + [TestGroup(TestType = TestType.System)] + public class SpreaderTests : IDisposable { + ServiceContext _context; + OrderService _order_service; + AdminService _admin_service; + MarketDataRelay _md_relay; + IDisposable _order_subscription; + Update _orders; + public SpreaderTests (){ + _context = new ServiceContext(); + + _order_service = _context.locate(); + _admin_service = _context.locate(); + _md_relay = _context.locate(); + + var auth = _context.login("bts",""); + auth.Wait(3000); + + _order_subscription = _order_service.subscribei_orders().Subscribe(ords => + { + _orders = ords; + }); + } + + public void Dispose() { + Disposable.dispose(ref _order_service); + Disposable.dispose(ref _admin_service); + Disposable.dispose(ref _context); + Disposable.dispose(ref _order_subscription); + } + + [Test(Timeout = 30)] + public async Task spreader_test() { + bool system_ready = await TestUtils.test_for_system_ready(_context, TimeSpan.FromMinutes(5)); + Assert.IsTrue(system_ready, "System is not ready"); + + string test_symbol = "BZ"; + var test_user = (name: "bts", id: 1u); + var test_account = (name: "ACC A", id: 1u); + + var test_securities = await SpreaderUtilities.get_securities_for_common_setup_one(_md_relay, test_symbol); + + await SpreaderUtilities.common_setup_one(_order_service, test_securities, test_account, test_user); + + } + + } +} \ No newline at end of file diff --git a/src/client/spark_test/Utils/MarketDataUtilities.cs b/src/client/spark_test/Utils/MarketDataUtilities.cs index b7b9c9781..5ab58171a 100644 --- a/src/client/spark_test/Utils/MarketDataUtilities.cs +++ b/src/client/spark_test/Utils/MarketDataUtilities.cs @@ -39,5 +39,36 @@ namespace bts.tests { return null; } + public static async Task> get_expirations(MarketDataRelay relay, string symbol, SecurityType security_type, MDRequest.Types.Flags flags = MDRequest.Types.Flags.Bbo, Venue v = Venue.Sim) { + var request = new MDRequest() { + Host = _MD_HOST, + Channels = { + new SecurityChannel() { + Time = 0, + Flags = (uint)flags, + Type = security_type, + Exchange = v, + Symbol = symbol, + TradeCursor = 0, + RfqCursor = 0 + } + } + }; + + var result = await relay.requestmd_async(request); + + var futures = new List(); + + if (result.success) { + foreach (var ch in result.value.Channels) { + foreach (var update in ch.Updates) { + futures.Add(update.Security.Future); + } + } + } + + return (IEnumerable) futures; + } + } } diff --git a/src/client/spark_test/Utils/OrderUtilities.cs b/src/client/spark_test/Utils/OrderUtilities.cs index b64bc5662..906eb013e 100644 --- a/src/client/spark_test/Utils/OrderUtilities.cs +++ b/src/client/spark_test/Utils/OrderUtilities.cs @@ -61,7 +61,7 @@ namespace bts.tests } } - public async static Task>> + public async static Task>> wait_for_filtered_orders(OrderService order_service, Func, bool> match, Func, IEnumerable> selector, TimeSpan? timeout = null) { if (timeout == null) { timeout = TimeSpan.FromSeconds(3); @@ -93,6 +93,33 @@ namespace bts.tests return tr; } + public async static Task> create_book(OrderService order_service, (OrderDefinition order_def, double price, uint size)[] order_defs) { + + List> results = new List>(); + + foreach(var ord in order_defs) { + Assert.IsTrue(ord.order_def != null, "Order definition can not be null"); + results.Add(await send_order(order_service, ord.order_def, ord.price, ord.size)); + } + + Func, IEnumerable> selector = existing_orders => { + return existing_orders.Where(z => results.Exists(y => z.Key == y.value.OrderKey)); + }; + + Func, bool> match = existing_orders => { + return selector(existing_orders).Count() > 0; + }; + + await Task.Delay(5000);// I'm not sure how to get around this + + utils.TestUtils.TimeoutResult> filtered_orders = await wait_for_filtered_orders(order_service, match, selector, TimeSpan.FromSeconds(30)); + + Assert.IsNotNull(filtered_orders.result, "Book is incomplete"); + Assert.IsTrue(filtered_orders.result.Count() == order_defs.Length, "Book is incomplete"); + + return filtered_orders.result; + } + } /// diff --git a/src/client/spark_test/Utils/SpreaderUtilities.cs b/src/client/spark_test/Utils/SpreaderUtilities.cs new file mode 100644 index 000000000..638dee19e --- /dev/null +++ b/src/client/spark_test/Utils/SpreaderUtilities.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive.Linq; +using System.Text; +using System.Threading.Tasks; +using bts.api; +using bts.api.admin; +using bts.api.core; +using bts.api.test; +using bts.test; +using bts.api.orderman; +using mbts; +using mbts.tests; +using bts.co2; +using bts.tests.utils; + +namespace bts.tests +{ + public class SpreaderUtilities { + + public static async Task<(SecurityImmutable s1, SecurityImmutable s2)> get_securities_for_common_setup_one(MarketDataRelay relay, string symbol) { + var exps = (await MarketDataUtilities.get_expirations(relay, symbol, SecurityType.Future)).ToList(); + + Assert.IsNotNull(exps); + Assert.IsTrue(exps.Count() > 2, "Need at least two expirations"); + + var exp1 = exps[0]; + var exp2 = exps[1]; + + Func sec1_match = ch => + ch.Security.Future.ExpirationYear == exp1.ExpirationYear && + ch.Security.Future.ExpirationMonth == exp1.ExpirationMonth && + ch.Security.Future.ExpirationDay == exp1.ExpirationDay; + + Func sec2_match = ch => + ch.Security.Future.ExpirationYear == exp2.ExpirationYear && + ch.Security.Future.ExpirationMonth == exp2.ExpirationMonth && + ch.Security.Future.ExpirationDay == exp2.ExpirationDay; + + var sec_task1 = MarketDataUtilities.get_security(relay, symbol, SecurityType.Future, sec1_match); + var sec_task2 = MarketDataUtilities.get_security(relay, symbol, SecurityType.Future, sec2_match); + + var sec1 = await sec_task1; + var sec2 = await sec_task2; + + Assert.IsFalse(sec1 == null, "security is null"); + Assert.IsFalse(sec2 == null, "security is null"); + + return (s1: sec1.Value, s2: sec2.Value); + } + public static async Task common_setup_one(OrderService order_service, (SecurityImmutable? s1, SecurityImmutable? s2) securities, (String name, uint id) account, (String name, uint id) user) { + var order_defs = new (OrderDefinition order_def, double price, uint size)[10]; + + for (int i = 0; i < 4; i ++) { + double p = (0.01 * i) + 18.230; + order_defs[i] = ( + order_def: OrderDefinitionUtilities.create_limit_order_def(securities.s1.Value.Id, Side.Buy, account, user), + price: p, + size: 1 + ); + } + + for (int i = 4; i < 8; i ++) { + double p = (0.01 * (i - 4)) + 18.300; + order_defs[i] = ( + order_def: OrderDefinitionUtilities.create_limit_order_def(securities.s1.Value.Id, Side.Sell, account, user), + price: p, + size: 1 + ); + } + + order_defs[8] = ( + order_def: OrderDefinitionUtilities.create_limit_order_def(securities.s2.Value.Id, Side.Buy, account, user), + price: 18.500, + size: 5 + ); + + order_defs[9] = ( + order_def: OrderDefinitionUtilities.create_limit_order_def(securities.s2.Value.Id, Side.Sell, account, user), + price: 18.53, + size: 5 + ); + + var book = await OrderUtilities.create_book(order_service, order_defs); + + } + } +} \ No newline at end of file From 3171ae0afb2dd9c3b6df5c3b130d6d611baab77f Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Thu, 11 Apr 2019 16:06:59 -0400 Subject: [PATCH 28/85] Create a Synthetic Security --- src/client/spark_test/System/SpreaderTests.cs | 11 ++- .../spark_test/Utils/SpreaderUtilities.cs | 91 ++++++++++++++++++- .../system_test_scripts/spreader_tests.sh | 2 +- 3 files changed, 99 insertions(+), 5 deletions(-) diff --git a/src/client/spark_test/System/SpreaderTests.cs b/src/client/spark_test/System/SpreaderTests.cs index 03d0cde7f..29e7160cb 100644 --- a/src/client/spark_test/System/SpreaderTests.cs +++ b/src/client/spark_test/System/SpreaderTests.cs @@ -14,6 +14,8 @@ using mbts; using mbts.tests; using bts.co2; using bts.tests.utils; +using bts.api.synthetic; +using bts.spark; namespace bts.tests { @@ -23,6 +25,7 @@ namespace bts.tests OrderService _order_service; AdminService _admin_service; MarketDataRelay _md_relay; + SyntheticSecurityService _synthetic_service; IDisposable _order_subscription; Update _orders; public SpreaderTests (){ @@ -31,6 +34,7 @@ namespace bts.tests _order_service = _context.locate(); _admin_service = _context.locate(); _md_relay = _context.locate(); + _synthetic_service = _context.locate(); var auth = _context.login("bts",""); auth.Wait(3000); @@ -49,7 +53,7 @@ namespace bts.tests } [Test(Timeout = 30)] - public async Task spreader_test() { + public async Task spreader_simple_sanity() { bool system_ready = await TestUtils.test_for_system_ready(_context, TimeSpan.FromMinutes(5)); Assert.IsTrue(system_ready, "System is not ready"); @@ -57,9 +61,10 @@ namespace bts.tests var test_user = (name: "bts", id: 1u); var test_account = (name: "ACC A", id: 1u); - var test_securities = await SpreaderUtilities.get_securities_for_common_setup_one(_md_relay, test_symbol); + (SecurityImmutable s1, SecurityImmutable s2) test_securities = await SpreaderUtilities.get_securities_for_common_setup_one(_md_relay, test_symbol); + + var setup_product = await SpreaderUtilities.common_setup_one(_order_service, _synthetic_service, test_securities, test_account, test_user); - await SpreaderUtilities.common_setup_one(_order_service, test_securities, test_account, test_user); } diff --git a/src/client/spark_test/Utils/SpreaderUtilities.cs b/src/client/spark_test/Utils/SpreaderUtilities.cs index 638dee19e..bb75d22f5 100644 --- a/src/client/spark_test/Utils/SpreaderUtilities.cs +++ b/src/client/spark_test/Utils/SpreaderUtilities.cs @@ -14,6 +14,8 @@ using mbts; using mbts.tests; using bts.co2; using bts.tests.utils; +using bts.api.synthetic; +using bts.spark; namespace bts.tests { @@ -49,7 +51,7 @@ namespace bts.tests return (s1: sec1.Value, s2: sec2.Value); } - public static async Task common_setup_one(OrderService order_service, (SecurityImmutable? s1, SecurityImmutable? s2) securities, (String name, uint id) account, (String name, uint id) user) { + public static async Task<(IEnumerable book, uint? synthetic_id)> common_setup_one(OrderService order_service, SyntheticSecurityService synthetic_service, (SecurityImmutable? s1, SecurityImmutable? s2) securities, (String name, uint id) account, (String name, uint id) user) { var order_defs = new (OrderDefinition order_def, double price, uint size)[10]; for (int i = 0; i < 4; i ++) { @@ -84,6 +86,93 @@ namespace bts.tests var book = await OrderUtilities.create_book(order_service, order_defs); + List<(ContractLeg contract, LegSpreaderParams parameters)> legs = new List<(ContractLeg, LegSpreaderParams)>(); + + legs.Add(create_leg(securities.s1.Value.CloneToMutable(),1,1,false,1)); + legs.Add(create_leg(securities.s2.Value.CloneToMutable(),-1,-1,true,1)); + + uint? synthetic_id = await create_synthetic(synthetic_service, legs); + + return(book, synthetic_id); + + } + + public static async Task create_synthetic(SyntheticSecurityService service, IEnumerable<(ContractLeg contract, LegSpreaderParams parameters)> legs) { + var new_synth = new Security(){}; + new_synth.Synthetic = new SyntheticContract(); + new_synth.Synthetic.Legs.Add(legs.Select(l => l.contract)); + new_synth.Symbol = deduce_symbol(new_synth); + new_synth.ContractSpec = new ContractSpecification(); + new_synth.ContractSpec.LowTick = 0.01; + new_synth.ContractSpec.HighTick = 0.01; + new_synth.ContractSpec.PriceBreak = double.MaxValue; + new_synth.ContractSpec.MainFraction = 1u; + new_synth.ContractSpec.PriceDisplayFormat = 3u; + new_synth.Exchange = Venue.Bts; + new_synth.Synthetic.Name = "TEST_SYNTH"; + + var result = await service.create_synthetic_async(new_synth); + Assert.IsTrue(result.success, "Unable to create synthetic"); + Assert.IsTrue(result.value.Error == CreateResult.Types.Error.Ok, "Create synthetic was not OK"); + + uint? id = result.value.Key; + Assert.IsTrue(id.HasValue && id.Value != 0, "Failed to get synthetic security key"); + + ExecutionStrategy strategy = new ExecutionStrategy(); + foreach(var leg in legs){ + strategy.LegParamsMap.Add(leg.contract.Security.Id, leg.parameters); + } + bool success = await save_strategy(service, id.Value, new ExecutionStrategyImmutable(strategy)); + Assert.IsTrue(success, "Failed to save synthetic strategy"); + + return id; + + } + + public static (ContractLeg contract, LegSpreaderParams parameters) create_leg( + Security security, + double quantity, + double price_multiplier, + bool is_quoting, + double lean_qty_factor, + LegSpreaderParams.Types.MarketDataType market_data_type = 0 + ) + { + var contract = new ContractLeg(); + contract.Security = security; + contract.PriceMultiplier = price_multiplier; + contract.Side = quantity > 0 ? Side.Buy : Side.Sell; + contract.Quantity = Math.Abs(quantity); + + var parameters = new LegSpreaderParams(); + parameters.IsActiveQuoting = is_quoting; + parameters.MarketDataType = market_data_type; // 0 == Merged, 1 == Direct + parameters.EnableHedging = !is_quoting; + parameters.PostHedgeParams = new PostHedgeParams(); + parameters.PreHedgeParams = new PreHedgeParams(); + + parameters.QuoteParams = new QuoteParams(); + parameters.QuoteParams.LeanQuantityRatio = lean_qty_factor; + parameters.QuoteParams.QuoteMuteZone = null; // == Throttle unchecked + + return (contract: contract, parameters: parameters); + } + + public static string deduce_symbol(SecurityImmutable? security) { + if (!string.IsNullOrWhiteSpace(security?.Symbol)) return security?.Symbol; + if (security?.Synthetic.Legs.Count() > 0) return security?.Synthetic.Legs.First().Security.Symbol; + return string.Empty; + } + + public static async Task create_synthetic(SyntheticSecurityService service, SecurityImmutable? synthetic) { + var result = await service.create_synthetic_async(synthetic.Value); + Assert.IsTrue(result.success, "failed to create synthetic"); + return result.value.Key; + } + + public static async Task save_strategy(SyntheticSecurityService service, uint synthetic_id, ExecutionStrategyImmutable strategy) { + var result = await service.update_security_metadata_async(new Metadata { SecurityKey = synthetic_id, Strategy = strategy.CloneToMutable() }); + return result.success; } } } \ No newline at end of file diff --git a/src/client/spark_test/system_test_scripts/spreader_tests.sh b/src/client/spark_test/system_test_scripts/spreader_tests.sh index 10fad4d10..01f2a03de 100644 --- a/src/client/spark_test/system_test_scripts/spreader_tests.sh +++ b/src/client/spark_test/system_test_scripts/spreader_tests.sh @@ -26,7 +26,7 @@ echo "sparkd pid: " $pid echo "sleeping for 120 seconds..." sleep 120 -dotnet sparkcmd.dll test spreader_test +dotnet sparkcmd.dll test spreader_simple_sanity kill $pid mv ~/sparkdata/conf/spark.conf.bak ~/sparkdata/conf/spark.conf \ No newline at end of file From 6ad12e922c8151eda58a3b6461ffe6f59ed995dc Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Tue, 16 Apr 2019 13:03:36 -0400 Subject: [PATCH 29/85] Submit and confirm working spreader order --- src/client/spark_test/System/SpreaderTests.cs | 17 +++++++++++++++++ src/client/spark_test/Utils/OrderUtilities.cs | 7 +++++++ .../spark_test/Utils/SpreaderUtilities.cs | 16 +++++++++------- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/client/spark_test/System/SpreaderTests.cs b/src/client/spark_test/System/SpreaderTests.cs index 29e7160cb..5996d2842 100644 --- a/src/client/spark_test/System/SpreaderTests.cs +++ b/src/client/spark_test/System/SpreaderTests.cs @@ -65,7 +65,24 @@ namespace bts.tests var setup_product = await SpreaderUtilities.common_setup_one(_order_service, _synthetic_service, test_securities, test_account, test_user); + var order1_def = OrderDefinitionUtilities.create_spreader_order_def( + setup_product.synthetic.security, + Side.Buy, + setup_product.synthetic.strategy, + test_account, + test_user + ); + var order1 = await OrderUtilities.send_order(_order_service, order1_def, -0.30, 1); + var spreader_orders = await OrderUtilities.wait_for_filtered_orders(_order_service, + o => o.Where(ordi => ordi.Key == order1.value.OrderKey).Count() > 0, + ord => ord.Where(ordi => ordi.Key == order1.value.OrderKey)); + + Assert.IsTrue(spreader_orders.result.Count() > 0, "Can't find spreader order"); + + var order = spreader_orders.result.TryFirst(); + Assert.IsTrue(order.HasValue, "order not found"); + Assert.IsTrue(order.Value.State.Status == OrderStatus.Working, "Order is not Working"); } } diff --git a/src/client/spark_test/Utils/OrderUtilities.cs b/src/client/spark_test/Utils/OrderUtilities.cs index 906eb013e..031484a5c 100644 --- a/src/client/spark_test/Utils/OrderUtilities.cs +++ b/src/client/spark_test/Utils/OrderUtilities.cs @@ -14,6 +14,7 @@ using mbts; using mbts.tests; using bts.co2; using bts.tests.utils; +using bts.spark; namespace bts.tests { @@ -147,6 +148,12 @@ namespace bts.tests return def; } + public static OrderDefinition create_spreader_order_def(Security security, Side side, ExecutionStrategy strategy, (String name, uint id) account, (String name, uint id) user) { + var def = create_base_order(security.Id, side, account, user); + def.CustomDef = new CustomOrderDefinition { Spreader = new SpreaderOrderDefinition {Security = security, ExecutionStrategy = strategy} }; + return def; + } + public static OrderDefinition create_iceberg_order_def(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user, double show_quantity) { var def = create_base_order(security_id, side, account, user); def.CustomDef = new CustomOrderDefinition { diff --git a/src/client/spark_test/Utils/SpreaderUtilities.cs b/src/client/spark_test/Utils/SpreaderUtilities.cs index bb75d22f5..db2f830f6 100644 --- a/src/client/spark_test/Utils/SpreaderUtilities.cs +++ b/src/client/spark_test/Utils/SpreaderUtilities.cs @@ -51,7 +51,7 @@ namespace bts.tests return (s1: sec1.Value, s2: sec2.Value); } - public static async Task<(IEnumerable book, uint? synthetic_id)> common_setup_one(OrderService order_service, SyntheticSecurityService synthetic_service, (SecurityImmutable? s1, SecurityImmutable? s2) securities, (String name, uint id) account, (String name, uint id) user) { + public static async Task<(IEnumerable book, (Security security, ExecutionStrategy strategy) synthetic)> common_setup_one(OrderService order_service, SyntheticSecurityService synthetic_service, (SecurityImmutable? s1, SecurityImmutable? s2) securities, (String name, uint id) account, (String name, uint id) user) { var order_defs = new (OrderDefinition order_def, double price, uint size)[10]; for (int i = 0; i < 4; i ++) { @@ -91,13 +91,13 @@ namespace bts.tests legs.Add(create_leg(securities.s1.Value.CloneToMutable(),1,1,false,1)); legs.Add(create_leg(securities.s2.Value.CloneToMutable(),-1,-1,true,1)); - uint? synthetic_id = await create_synthetic(synthetic_service, legs); + (Security security, ExecutionStrategy strategy) synthetic = await create_synthetic(synthetic_service, legs); - return(book, synthetic_id); + return(book, synthetic); } - public static async Task create_synthetic(SyntheticSecurityService service, IEnumerable<(ContractLeg contract, LegSpreaderParams parameters)> legs) { + public static async Task<(Security, ExecutionStrategy)> create_synthetic(SyntheticSecurityService service, IEnumerable<(ContractLeg contract, LegSpreaderParams parameters)> legs) { var new_synth = new Security(){}; new_synth.Synthetic = new SyntheticContract(); new_synth.Synthetic.Legs.Add(legs.Select(l => l.contract)); @@ -109,7 +109,7 @@ namespace bts.tests new_synth.ContractSpec.MainFraction = 1u; new_synth.ContractSpec.PriceDisplayFormat = 3u; new_synth.Exchange = Venue.Bts; - new_synth.Synthetic.Name = "TEST_SYNTH"; + new_synth.Synthetic.Name = "TEST_SYNTH" + DateTime.Now.ToString(); var result = await service.create_synthetic_async(new_synth); Assert.IsTrue(result.success, "Unable to create synthetic"); @@ -118,14 +118,16 @@ namespace bts.tests uint? id = result.value.Key; Assert.IsTrue(id.HasValue && id.Value != 0, "Failed to get synthetic security key"); + new_synth.Id = result.value.Key; + ExecutionStrategy strategy = new ExecutionStrategy(); foreach(var leg in legs){ strategy.LegParamsMap.Add(leg.contract.Security.Id, leg.parameters); } - bool success = await save_strategy(service, id.Value, new ExecutionStrategyImmutable(strategy)); + bool success = await save_strategy(service, new_synth.Id, new ExecutionStrategyImmutable(strategy)); Assert.IsTrue(success, "Failed to save synthetic strategy"); - return id; + return (new_synth, strategy); } From bd643aa525661dff8b71994e00dd2a8afe5c2f69 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Fri, 26 Apr 2019 14:50:16 -0500 Subject: [PATCH 30/85] don't draw piq on every tick; might save some cycles --- .../spark/view/Spark/Ladder/LadderGridHost.cs | 19 ++++++++++---- .../spark/view/Spark/Ladder/LadderOrderVM.cs | 26 ++++++++++++++++++- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderGridHost.cs b/src/client/spark/view/Spark/Ladder/LadderGridHost.cs index 516ccf0f8..4c04913ba 100644 --- a/src/client/spark/view/Spark/Ladder/LadderGridHost.cs +++ b/src/client/spark/view/Spark/Ladder/LadderGridHost.cs @@ -55,7 +55,7 @@ namespace mbts.spark { LadderTactileAdorner _tactile_adorner; ulong _prices_updated; NativePriceSize[] _bids, _asks; - bool _price_dirty = false; + bool _price_dirty, _piq_dirty = false; double[] _price_list = null; IComparer _price_comparer; double _last_origin_price; @@ -235,11 +235,15 @@ namespace mbts.spark { case NotifyCollectionChangedAction.Add: foreach (LadderOrderVM item in e.NewItems) { _ordershost.add_order(item); + item.piq_changed += _set_piq_dirty; + _set_piq_dirty(); } break; case NotifyCollectionChangedAction.Remove: foreach (LadderOrderVM item in e.OldItems) { _ordershost.remove_order(item); + item.piq_changed -= _set_piq_dirty; + _set_piq_dirty(); } break; case NotifyCollectionChangedAction.Reset: @@ -250,6 +254,9 @@ namespace mbts.spark { throw new NotImplementedException(); } } + + void _set_piq_dirty() { _piq_dirty = true; } + #endregion Column[] columns { get; set; } = new Column[0]; @@ -438,6 +445,7 @@ namespace mbts.spark { _prices_updated = 0; _price_comparer = Comparer.Create((x, y) => security.ContractSpec.tick_equals(x, y) ? 0 : y.CompareTo(x)); _price_dirty = true; + _piq_dirty = true; _vm.on_tick += _tick; _settings_sub = _vm.settings.OnDispatcher().Subscribe(_settings_updated); _volume_sub = _vm.model.md_auxiliary_model.security_volume_at_price(security.Id).Subscribe(t => _volume_container.write_value(t.state.Values)); @@ -523,8 +531,10 @@ namespace mbts.spark { if (depth_changed) _refresh_sizes(); } - // TODO - detect changes in PIQ to selectively redraw - _refresh_piq(); + if (_piq_dirty) { + _refresh_piq(); + _piq_dirty = false; + } _refresh_volume_at_price(); _last_origin_price = _mkt_break_lines.origin; _price_dirty = false; @@ -565,8 +575,7 @@ namespace mbts.spark { if (!_settings.piq_column.show) return; int origin_index = LadderRender.index_of(_price_list, _mkt_break_lines.origin, _price_comparer); - var piq_list = _vm.price_vs_piq; - LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, origin_index, _price_list, piq_list.ToArray(), _settings.row_height, security.ContractSpec, _settings.piq_column); + LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, origin_index, _price_list, _vm.price_vs_piq.ToArray(), _settings.row_height, security.ContractSpec, _settings.piq_column); } void _refresh_sizes() { diff --git a/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs b/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs index a78ff0443..f4b173b3d 100644 --- a/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderOrderVM.cs @@ -44,6 +44,9 @@ namespace mbts.spark { double _real_price = double.NaN, _phantom_price = double.NaN; uint filled, quantity; int working_quantity; + uint? _min_piq = UInt32.MaxValue; + + public event Action piq_changed; LadderModel model => _parent.model; Dictionary _orders = new Dictionary(); @@ -134,10 +137,22 @@ namespace mbts.spark { public void add_order((OrderRecordImmutable order, uint? p) new_op) { _orders[new_op.order.Key] = new_op; + // reevaluate min piq if this order's piq is less + if (new_op.p.HasValue) { + if (new_op.p.Value < min_piq) { + min_piq = new_op.p.Value; + } + } } public void remove_order(uint order_key) { _orders.Remove(order_key); + // on a remove, potentially the order w/ the lowest piq has been removed, so + // if the min() is now > than the current min_piq, update the min_piq + var prospective_piq = _orders.Select(op => op.Value.p).Where(piq => piq.HasValue).Min(); + if (prospective_piq > min_piq) { + min_piq = prospective_piq; + } } //public void update(OrderRecordImmutable neworder) { @@ -149,7 +164,16 @@ namespace mbts.spark { public uint security_key => _parent.security.Id; public double price { get => double.IsNaN(_phantom_price) ? _real_price : _phantom_price; } - public uint? min_piq => _orders.Select(op => op.Value.p).Where(piq => piq.HasValue).Min(); + public uint? min_piq { + get { + return _min_piq; + } + set { + if (Set(ref _min_piq, value)) { + piq_changed?.Invoke(); + } + } + } public double real_price => _real_price; public Side side { get { From 77f57239368d55a3c41ce0ebbae7abf594c5673b Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 30 Apr 2019 12:23:18 -0400 Subject: [PATCH 31/85] Apply lazyView to the limits tables --- src/web/ui/src/Accounts/View.fs | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/web/ui/src/Accounts/View.fs b/src/web/ui/src/Accounts/View.fs index 547808baf..be6133fbf 100644 --- a/src/web/ui/src/Accounts/View.fs +++ b/src/web/ui/src/Accounts/View.fs @@ -10,6 +10,7 @@ open EnumUtils open Fable open System open bts.co2 +open Elmish.React.Common let accountIsSelected (initialState:BTSAccount) (scratch:BTSAccount) = scratch <> initialState @@ -179,7 +180,6 @@ let getSecurityMap (model:Model) (positionLimit:AccountSecurityPositionLimitUnit let clipLimitsView model dispatch = - let symbols = GetSymbolArray model let headerRow:TableControl.row = { cells = @@ -322,7 +322,8 @@ let clipLimitsView model dispatch = | true -> "is-selected" header = None } - ] @ [footerRow] + yield footerRow + ] let tempSec = { SecurityScope.empty with securitySymbol = match GetFilteredSymbolArray model usedSymbolNames |> Array.tryHead with @@ -347,6 +348,13 @@ let clipLimitsView model dispatch = TableControl.createDataTable (Some headerRow) rws (Button.button "Add" "" "plus-circle" Button.Enabled addRow) ] +let lazyClipLimitsView model dispatch = + lazyView2With (fun a b -> + LanguagePrimitives.PhysicalEquality a.SelectedClipLimits b.SelectedClipLimits && + LanguagePrimitives.PhysicalEquality a.scratch.securityClipLimits b.scratch.securityClipLimits) + clipLimitsView + model dispatch + let productLimits model dispatch = let headerRow:TableControl.row = { @@ -534,6 +542,12 @@ let productLimits model dispatch = br [] ] +let lazyProductLimitsView model dispatch = + lazyView2With (fun a b -> + LanguagePrimitives.PhysicalEquality a.SelectedPositionLimits b.SelectedPositionLimits && + LanguagePrimitives.PhysicalEquality a.scratch.productPositionLimits b.scratch.productPositionLimits) + productLimits + model dispatch let securityLimitsView model dispatch = let headerRow:TableControl.row = @@ -653,8 +667,9 @@ let securityLimitsView model dispatch = let LimitsView model dispatch = div [ Class "limits-container"] [ - clipLimitsView model dispatch - productLimits model dispatch + lazyClipLimitsView model dispatch + lazyProductLimitsView model dispatch + //securityLimitsView model dispatch accountControls model dispatch From 8dde9bf3c558c2abbc022e637036c2c8b3a33f0a Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Wed, 1 May 2019 08:39:37 -0500 Subject: [PATCH 32/85] print descriptive workspace info after selecting a user's workspace to copy --- .../view/Shell/Workspaces/SettingsManagerTool.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/client/spark/view/Shell/Workspaces/SettingsManagerTool.cs b/src/client/spark/view/Shell/Workspaces/SettingsManagerTool.cs index d25162176..d8e5d5a97 100644 --- a/src/client/spark/view/Shell/Workspaces/SettingsManagerTool.cs +++ b/src/client/spark/view/Shell/Workspaces/SettingsManagerTool.cs @@ -19,6 +19,21 @@ namespace bts.settings { var from_persist = new UserUISettings(user_list.Users.First(u => u.Login == from).Id, persist, null); var workspaces = from_persist.query(WorkspacesModel.WORKSPACE_NAMESPACE, ".*").Result; + + ConsoleEx.WriteLine("Workspace Descriptions", ConsoleColor.Cyan); + foreach(var w in workspaces) { + var r = from ws in w.value.windows + group ws by ws.owner into grouping + select new { key = grouping.Key.Replace("bts.shell.", ""), window_count = grouping.Count() }; + string desc = string.Empty; + foreach(var i in r) { + desc += $"{Environment.NewLine}\t{i.window_count} - {i.key}"; + } + ConsoleEx.Write($"{w.key}", ConsoleColor.Yellow); + ConsoleEx.WriteLine($" - {w.value.windows.Count} windows - {w.value.named_windows.Count} named windows - {desc}"); + } + ConsoleEx.WriteLine(""); + var selected_ws = ConsoleEx.prompt("Choose workspace:", workspaces.Select(w => w.key)); var to = ConsoleEx.prompt("To User", user_names); From 6d9c083c0fb3b82607d0062b8aa1287c4dc17955 Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Thu, 2 May 2019 10:58:45 -0400 Subject: [PATCH 33/85] Verify quoting order --- src/client/spark_test/System/SpreaderTests.cs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/client/spark_test/System/SpreaderTests.cs b/src/client/spark_test/System/SpreaderTests.cs index 5996d2842..90fa8123f 100644 --- a/src/client/spark_test/System/SpreaderTests.cs +++ b/src/client/spark_test/System/SpreaderTests.cs @@ -57,7 +57,7 @@ namespace bts.tests bool system_ready = await TestUtils.test_for_system_ready(_context, TimeSpan.FromMinutes(5)); Assert.IsTrue(system_ready, "System is not ready"); - string test_symbol = "BZ"; + string test_symbol = "SI";// don't use ES, the prices will be rounded to the same number and the result will be off var test_user = (name: "bts", id: 1u); var test_account = (name: "ACC A", id: 1u); @@ -77,12 +77,22 @@ namespace bts.tests var spreader_orders = await OrderUtilities.wait_for_filtered_orders(_order_service, o => o.Where(ordi => ordi.Key == order1.value.OrderKey).Count() > 0, ord => ord.Where(ordi => ordi.Key == order1.value.OrderKey)); - Assert.IsTrue(spreader_orders.result.Count() > 0, "Can't find spreader order"); - var order = spreader_orders.result.TryFirst(); + var order = spreader_orders.result.Where(ordi => ordi.Key == order1.value.OrderKey).TryFirst(); Assert.IsTrue(order.HasValue, "order not found"); Assert.IsTrue(order.Value.State.Status == OrderStatus.Working, "Order is not Working"); + + //Verify that a quoting order of 18.60 shows up in the second leg. + var quoting_orders = await OrderUtilities.wait_for_filtered_orders(_order_service, + o => o.Where(ordi => ordi.ParentKey == order1.value.OrderKey).Count() > 0, + ord => ord.Where(ordi => ordi.ParentKey == order1.value.OrderKey)); + + var quoting_order = quoting_orders.result.TryFirst(); + Assert.IsTrue(quoting_order.HasValue, "order not found"); + Assert.IsTrue(quoting_order.Value.State.Status == OrderStatus.Working, "Order is not Working"); + Assert.IsTrue(quoting_order.Value.State.Price == 18.6, "Quoting order has the wrong price"); + } } From 102fefbd925bd23b98cafb16a3abd8e4b7a6cc08 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 3 May 2019 13:13:59 -0400 Subject: [PATCH 34/85] 1.6.2 release notes --- doc/user/content/release_notes/_index.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/user/content/release_notes/_index.md b/doc/user/content/release_notes/_index.md index 3c158addf..f37e8d038 100644 --- a/doc/user/content/release_notes/_index.md +++ b/doc/user/content/release_notes/_index.md @@ -4,6 +4,11 @@ weight = 3 sort_by = "weight" insert_anchor_links = "right" +++ +## 1.6.2 +### User Visible Changes +### Exchange simulator + - Exchange simulator now now produce position in queue data + ## 1.6.0 ### User Visible Changes #### New Features From 84aefabfea60cef6831171d2f80f38d8875640f0 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 3 May 2019 13:19:35 -0400 Subject: [PATCH 35/85] Safe workaround for SPARK-2303. --- src/lib/md/relay/RelayClient.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/md/relay/RelayClient.cpp b/src/lib/md/relay/RelayClient.cpp index 9830a6cc7..54c3a9066 100644 --- a/src/lib/md/relay/RelayClient.cpp +++ b/src/lib/md/relay/RelayClient.cpp @@ -143,7 +143,9 @@ public: LOG(md,Diag, "Added {} trades", ch.trade_log_size()); } for (const auto& trade : ch.trade_log()) { - channel.add_trade(trade); + if (trade.key()) { + channel.add_trade(trade); + } } for (const auto& rfq : ch.rfq_log()) { channel.add_rfq(rfq); From 9f406e83aa1f64b221497ff097bfe193b93ea54d Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 3 May 2019 17:39:47 -0400 Subject: [PATCH 36/85] SPARK-2151 Dump shared memory on bad handle for forensic analysis --- src/client/spark_api/Co2/Co2NativeMethods.cs | 4 +++ src/client/spark_api/Co2/MDConsumer.cs | 12 +++++-- src/client/spark_api/Co2/MDConsumerHandle.cs | 13 +++++-- src/lib/md/MDConsumer.cpp | 20 ++++++++++- src/lib/md/MDConsumer.h | 2 ++ src/lib/md/MDConsumerInterop.cpp | 6 ++++ src/lib/utils/SharedList.h | 36 +++++++++++++++++++- 7 files changed, 86 insertions(+), 7 deletions(-) diff --git a/src/client/spark_api/Co2/Co2NativeMethods.cs b/src/client/spark_api/Co2/Co2NativeMethods.cs index f744b9963..b2ae76044 100644 --- a/src/client/spark_api/Co2/Co2NativeMethods.cs +++ b/src/client/spark_api/Co2/Co2NativeMethods.cs @@ -67,6 +67,10 @@ namespace mbts.co2 [DllImport("libspark")] internal unsafe static extern void md_consumer_get_system_md_symbols(raw_usersymbol_callback cb); + + [DllImport("libspark")] + internal unsafe static extern void dump_co2_memory_to_disk(IntPtr consumer, string path); + // // Relay control diff --git a/src/client/spark_api/Co2/MDConsumer.cs b/src/client/spark_api/Co2/MDConsumer.cs index 439d65af0..3d92f6e69 100644 --- a/src/client/spark_api/Co2/MDConsumer.cs +++ b/src/client/spark_api/Co2/MDConsumer.cs @@ -37,7 +37,10 @@ namespace mbts.co2 unsafe void init() { _raw_sec_cb = (key, buf, len, handle) => { var security = Co2NativeMethods.deserialize(buf, len); - var next = new SecurityAndHandle() { security = security, handle = new MDConsumerHandle(handle) }; + var next = new SecurityAndHandle() { + security = security, + handle = new MDConsumerHandle(_consumer, security, handle) + }; _securities.Add(next); _on_security?.Invoke(new List(1) { next }); }; @@ -67,7 +70,7 @@ namespace mbts.co2 public MDConsumerHandle try_get_handle(security_key key) { Co2NativeMethods.BlitPointerTuple raw; if (Co2NativeMethods.md_consumer_get_handle(_consumer, key, out raw)) { - return new MDConsumerHandle(raw); + return new MDConsumerHandle(this._consumer, null, raw); } return null; } @@ -77,7 +80,10 @@ namespace mbts.co2 List result = new List(); Co2NativeMethods.raw_security_callback cb = (key, buf, len, handle) => { var security = Co2NativeMethods.deserialize(buf, len); - result.Add(new SecurityAndHandle() { security = new Security(security), handle = new MDConsumerHandle(handle) }); + result.Add( + new SecurityAndHandle() { + security = new Security(security), + handle = new MDConsumerHandle(_consumer, security, handle) }); }; Co2NativeMethods.md_consumer_walk_security_list(_consumer, cb); return result; diff --git a/src/client/spark_api/Co2/MDConsumerHandle.cs b/src/client/spark_api/Co2/MDConsumerHandle.cs index 4a12f2f99..7b03bdaee 100644 --- a/src/client/spark_api/Co2/MDConsumerHandle.cs +++ b/src/client/spark_api/Co2/MDConsumerHandle.cs @@ -156,13 +156,22 @@ namespace mbts.co2 { // // Internal - internal MDConsumerHandle(Co2NativeMethods.BlitPointerTuple tuple) { + internal MDConsumerHandle( + IntPtr consumer, + Security security, + Co2NativeMethods.BlitPointerTuple tuple) { _header = (ColdHeader*)tuple.cold_header.ToPointer(); _cold = (ColdEntry*)tuple.cold_data.ToPointer(); if (has_depth) { _hot_depth = (HotDepthEntry*)tuple.hot.ToPointer(); - Log.assert(_hot_depth->key == _cold->security_key); + if (_hot_depth->key != _cold->security_key) { + var temp_path = Path.Combine(Path.GetTempPath(), "btsshm"); + Co2NativeMethods.dump_co2_memory_to_disk(consumer, temp_path); + Log.assert( + _hot_depth->key == _cold->security_key, + $"Please attach the contents of '${temp_path}' to the bug report you are about to file. security:{security?.Id} cold:{_cold->security_key} hot:{_hot->key}"); + } } else { _hot = (HotEntry*)tuple.hot.ToPointer(); diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index a303c34ad..5969ec994 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -269,6 +269,16 @@ public: if (!_cold.is_initialized()) return true; return is_heart_beat_ok(cold_header()); } + + void dump_shm_to(const char* directory) const { + _cold.dump_to(directory); + if (_hot) { + _hot->dump_to(directory); + } + if (_hot_depth) { + _hot_depth->dump_to(directory); + } + } private: @@ -533,7 +543,7 @@ private: /////////////////////////////////////////////////////////////////////////////////////////////////// class MDConsumerImpl { typedef tbb::queuing_mutex Mutex; - Mutex _workmutex; + mutable Mutex _workmutex; vector> _instances; std::shared_ptr _sci; MBOReader* _mbo; @@ -910,6 +920,12 @@ public: return result; } + void dump_shm_to(const char* p) const { + //Mutex::scoped_lock lock(_workmutex); + for (auto& inst : _instances) { + inst->dump_shm_to(p); + } + } }; @@ -957,6 +973,8 @@ MDConsumerChannel MDConsumer::get_channel(const std::string& symbol, SecurityTyp MDConsumerChannel MDConsumer::get_channel(const UserSymbol& u) const { return _pimpl->get_channel(u.symbol(), u.type(), u.exchange()); } +void MDConsumer::dump_shm_to(const char* p) const { _pimpl->dump_shm_to(p); } + MDConsumerChannel::MDConsumerChannel(const shared_ptr& inst) : _pimpl(inst) {} int64_t MDConsumerChannel::read_trades(google::protobuf::RepeatedPtrField& trades, int64_t cursor) { return _pimpl->read_trades(trades, cursor); } diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index 103e8bb19..d197499a3 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -67,6 +67,8 @@ public: } uintptr_t id() const { return reinterpret_cast(_pimpl.get()); } + void dump_shm_to(const char* directory) const; + private: //BTS_NO_COPY(MDConsumer); COPY OK std::shared_ptr _pimpl; diff --git a/src/lib/md/MDConsumerInterop.cpp b/src/lib/md/MDConsumerInterop.cpp index 17a36b072..88972112d 100644 --- a/src/lib/md/MDConsumerInterop.cpp +++ b/src/lib/md/MDConsumerInterop.cpp @@ -177,5 +177,11 @@ CAPI void post_on_scan_security_thread(just_action a) { md::scan_securities_thread().post(a); } +CAPI void dump_co2_memory_to_disk(MDConsumer* consumer, const char* path) { + if (consumer) { + consumer->dump_shm_to(path); + } +} + }} //namespace #endif diff --git a/src/lib/utils/SharedList.h b/src/lib/utils/SharedList.h index 345055ebf..9a47e2701 100644 --- a/src/lib/utils/SharedList.h +++ b/src/lib/utils/SharedList.h @@ -15,8 +15,10 @@ #include #include #include +#include #include +#include namespace bts { namespace utils { @@ -238,6 +240,38 @@ public: offset = sizeof(SegmentHeader); } } + + void dump_to(const char* directory) const { + using namespace std; + boost::system::error_code _; + if (!boost::filesystem::exists(directory, _)) { + boost::filesystem::create_directories(directory, _); + } + + if (!boost::filesystem::is_directory(directory)) { + LOG(md,Error, "Error dumping memory to {}", std::string(directory)); + return; + } + + boost::filesystem::path root(directory); + + // OK to hold the lock for a while as we're typically crashing when this is called + tbb::queuing_mutex::scoped_lock lock(_mutex); + + for (size_t x = 0; x < _segments.size(); ++x) { + char segment_name[512]; + SharedList::get_segment_name(segment_name, BTS_ARRAYSIZE(segment_name), _name, x); + + // create/overwrite output file + std::ofstream writer( (root/segment_name).string(), ios::out | ios::app | ios::binary); + if (writer.is_open()) { + // writer just loves signed types + writer.write( + (const char*)_segments[x].address, + (std::streamsize)_segments[x].size); + } + } + } private: BTS_NO_COPY(SharedListReader); @@ -281,7 +315,7 @@ private: const std::string _name; THeader* _header; // null value indicates no memory segment - tbb::queuing_mutex _mutex; + mutable tbb::queuing_mutex _mutex; std::unique_ptr _shm; std::vector _segments; }; From 3beb82c95923225de10be217e43851d3f19cfd31 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 6 May 2019 09:01:45 -0400 Subject: [PATCH 37/85] Add comment about why dump_to_shm does'nt need to hold mutex --- src/lib/md/MDConsumer.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index 5969ec994..3a6610dae 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -543,7 +543,7 @@ private: /////////////////////////////////////////////////////////////////////////////////////////////////// class MDConsumerImpl { typedef tbb::queuing_mutex Mutex; - mutable Mutex _workmutex; + Mutex _workmutex; vector> _instances; std::shared_ptr _sci; MBOReader* _mbo; @@ -921,7 +921,10 @@ public: } void dump_shm_to(const char* p) const { - //Mutex::scoped_lock lock(_workmutex); + // To be safe we should lock _work_mutex -but- + // we need this function to diagnose problems happening in the on_security callback + // which holds the _work_mutex. Since anyone calling this function is going down + // anyway we as well live dangerously. for (auto& inst : _instances) { inst->dump_shm_to(p); } From fa8598adbdd025ebb4aa8efbf2f7c9aedad4fae8 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 7 May 2019 09:22:44 -0500 Subject: [PATCH 38/85] PR suggestion - explicitly set default value --- src/client/spark/view/Spark/Ladder/LadderGridHost.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderGridHost.cs b/src/client/spark/view/Spark/Ladder/LadderGridHost.cs index 6f699a222..18b5971cc 100644 --- a/src/client/spark/view/Spark/Ladder/LadderGridHost.cs +++ b/src/client/spark/view/Spark/Ladder/LadderGridHost.cs @@ -55,7 +55,7 @@ namespace mbts.spark { LadderTactileAdorner _tactile_adorner; ulong _prices_updated; NativePriceSize[] _bids, _asks; - bool _price_dirty, _piq_dirty = false; + bool _price_dirty = false, _piq_dirty = false; double[] _price_list = null; IComparer _price_comparer; double _last_origin_price; From 499072c94f2d345a5382b983e5d921d36b6ea5b6 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 7 May 2019 09:33:25 -0500 Subject: [PATCH 39/85] PR edit - we don't need an array here --- src/client/spark/view/Spark/Ladder/LadderGridHost.cs | 2 +- src/client/spark/view/Spark/Ladder/LaderRender.cs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderGridHost.cs b/src/client/spark/view/Spark/Ladder/LadderGridHost.cs index 18b5971cc..80fdf17cf 100644 --- a/src/client/spark/view/Spark/Ladder/LadderGridHost.cs +++ b/src/client/spark/view/Spark/Ladder/LadderGridHost.cs @@ -576,7 +576,7 @@ namespace mbts.spark { if (!_settings.piq_column.show) return; int origin_index = LadderRender.index_of(_price_list, _mkt_break_lines.origin, _price_comparer); - LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, origin_index, _price_list, _vm.price_vs_piq.ToArray(), _settings.row_height, security.ContractSpec, _settings.piq_column); + LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, origin_index, _price_list, _vm.price_vs_piq, _settings.row_height, security.ContractSpec, _settings.piq_column); } void _refresh_sizes() { diff --git a/src/client/spark/view/Spark/Ladder/LaderRender.cs b/src/client/spark/view/Spark/Ladder/LaderRender.cs index dfbc80d07..4e4a12d5a 100644 --- a/src/client/spark/view/Spark/Ladder/LaderRender.cs +++ b/src/client/spark/view/Spark/Ladder/LaderRender.cs @@ -278,22 +278,22 @@ namespace mbts.spark { } // this is basically a copy of redner_sizes up above - internal static void render_piqs(DrawingVisual dv, GlyphContext glyph_context, int origin_index, double[] prices, (double price, uint piq)[] piqs, double row_height, ContractSpecificationImmutable spec, LadderColumnSettings settings) { + internal static void render_piqs(DrawingVisual dv, GlyphContext glyph_context, int origin_index, double[] prices, IEnumerable<(double price, uint piq)> pvp, double row_height, ContractSpecificationImmutable spec, LadderColumnSettings settings) { using (var dc = dv.RenderOpen()) { using (_draw_with_clip(dc, settings, row_height, prices.Length)) { - if (!prices.within_range(origin_index) || piqs.is_null_or_empty()) + if (!prices.within_range(origin_index) || pvp.is_null_or_empty()) return; var foreground = settings.foreground.rgb_brush(); int curr_index = origin_index; double curr_price = prices[curr_index]; - for (int i = 0; i < piqs.Length; i++) { - double price = piqs[i].price; + foreach (var entry in pvp) { + double price = entry.price; // prices are in descending order, so get diff from curr_price - price int tick_diff = spec.tickdiff_unchecked(curr_price, price); curr_index += tick_diff; curr_price = price; if (curr_index >= 0 && curr_index < prices.Length) { - var gr = format_text(piqs[i].piq.ToString(), curr_index, row_height, settings, glyph_context); + var gr = format_text(entry.piq.ToString(), curr_index, row_height, settings, glyph_context); dc.DrawGlyphRun(foreground, gr); } else break; } From b24f8553219c52e1e564c046af1b8a0c079f6807 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Thu, 9 May 2019 08:20:36 -0500 Subject: [PATCH 40/85] misc refactoring after root algo branch merge; OrderBookVM parameter passing no longer necessary --- .../Modules/Trading/OrderBook/OrderBookColumns.cs | 4 +--- .../spark/Modules/Trading/OrderBook/OrderBookVM.cs | 12 ++++++------ .../OrderBook/Settings/OrderBookFilterStates.cs | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs index 760b8a52f..ca549d10e 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs @@ -65,11 +65,9 @@ namespace mbts.spark { partial class OrderBookVM { public class OrderBookColumnCollection : ObservableCollection, IValueFactory<(FilterStates filters, IReadOnlyList sorting)>, IDisposable { readonly List _active_comparers = new List(); - readonly OrderBookVM _owner; CompositeDisposable _resources = new CompositeDisposable(); - public OrderBookColumnCollection(IBus bus, OrdersModel model, IObservable<(IEnumerable added, IEnumerable removed)> order_securities, OrderBookVM owner) { - _owner = owner; + public OrderBookColumnCollection(IBus bus, OrdersModel model, IObservable<(IEnumerable added, IEnumerable removed)> order_securities) { // security Add(security_column = new SecurityColumn( DiscreteValuesFilterVM.Create( diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs index e3b184fe7..2da113487 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs @@ -87,7 +87,7 @@ namespace mbts.spark { init_format: AlgoFormats.Attachable, editing_permissions: AlgoEditingPermissions.AllowEditStructure | AlgoEditingPermissions.AllowEditValues)); _settings_manager = settings_manager; - _columns = new OrderBookColumnCollection(bus, model, model.on_order_securities.Select(x => (x.added.Values, x.removed.Values)), this); + _columns = new OrderBookColumnCollection(bus, model, model.on_order_securities.Select(x => (x.added.Values, x.removed.Values))); // we use an ObservableAppendCollection b/c the mini-ob needs a cancel column at the front columns = new ObservableAppendCollection(_columns); _row_collection = new OperableHierarchicalCollection( @@ -515,11 +515,11 @@ namespace mbts.spark { if (!new_sort.Equals(old_sort)) { /// Both filter and sort have changed, use to avoid syncing twice. using (_row_collection.defer_refresh()) { - _row_collection.filter = new_filter.build_predicate(this); + _row_collection.filter = new_filter.build_predicate(); _row_collection.comparer = new_sort.comparer; } } else { - _row_collection.filter = new_filter.build_predicate(this); + _row_collection.filter = new_filter.build_predicate(); } } else if (!new_sort.Equals(old_sort)) _row_collection.comparer = new_sort.comparer; @@ -590,7 +590,7 @@ namespace mbts.spark { var fd = new FilterData(grid_settings_manager.algo_tree_display_mode, _columns.create_filter_states()); if (!fd.Equals(_cached_filter_data)) { _cached_filter_data = fd; - _row_collection.filter = fd.build_predicate(this); + _row_collection.filter = fd.build_predicate(); } } @@ -900,8 +900,8 @@ namespace mbts.spark { this.column_filters = column_filters; } - public Func build_predicate(OrderBookVM owner) { - var pred = column_filters.build_predicate(owner); + public Func build_predicate() { + var pred = column_filters.build_predicate(); if (algo_tree_display_mode == AlgoTreeDisplayMode.RootAndLeafNodes) { if (pred == null) { pred = tree_disp_pred; diff --git a/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookFilterStates.cs b/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookFilterStates.cs index 30068f043..2e210f265 100644 --- a/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookFilterStates.cs +++ b/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookFilterStates.cs @@ -116,7 +116,7 @@ namespace bts.shell.orderbook { } static class FilterStatesExtensions { - public static Func build_predicate(this FilterStates fs, OrderBookVM owner) { + public static Func build_predicate(this FilterStates fs) { var predicates = new List>(); // security if (fs.security_criteria?.is_defined() ?? false) From f1e90394a01c00f0960f518f2c0daed33f283038 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Thu, 9 May 2019 08:24:44 -0500 Subject: [PATCH 41/85] restoring Root Algo column header after merge --- src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs index ca549d10e..32ef7d1a6 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs @@ -190,7 +190,8 @@ namespace mbts.spark { DiscreteValuesFilterVM.CreateTableUpdate( o => new AlgoDataVM(o.root_algo), o => AlgoID.from_algo_key(o.root_algo.key), - vm => AlgoID.from_algo_key(vm.data.key), out var root_algo_type_observer, out var root_algo_type_items_view))); + vm => AlgoID.from_algo_key(vm.data.key), out var root_algo_type_observer, out var root_algo_type_items_view), + DefaultColumnHeaders.RootAlgoType)); _resources.Add(algo_items_source.Subscribe(root_algo_type_observer)); root_algo_type_items_view.CustomSort = algo_items_comp; root_algo_type_items_view.GroupDescriptions.Add(algo_items_group_desc); From ff69cafe7f1c5ceda49823b0b39ea17cd3c42b01 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Thu, 9 May 2019 09:59:40 -0500 Subject: [PATCH 42/85] ob alignments - left aligning nearly everything, price and qty cols stay right aligned --- .../OrderBook/Settings/OrderBookSettings.cs | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookSettings.cs b/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookSettings.cs index 2be037993..d15c1c303 100644 --- a/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookSettings.cs +++ b/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookSettings.cs @@ -140,28 +140,28 @@ namespace bts.shell.orderbook { static class DefaultColumns { public static GridColumnSettings security => new GridColumnSettings(Columns.Security.ToString(), DIYGrid.DefaultColumnHeaders.Security, width: DIYGrid.DefaultColumnWidths.Security); - public static GridColumnSettings symbol => new GridColumnSettings(Columns.Symbol.ToString(), DIYGrid.DefaultColumnHeaders.Symbol, width: DIYGrid.DefaultColumnWidths.Symbol, horizontal_alignment: HorizontalAlignment.Right); - public static GridColumnSettings time_created => new GridColumnSettings(Columns.Time.ToString(), "Time", width: DIYGrid.DefaultColumnWidths.Time, horizontal_alignment: HorizontalAlignment.Right); - public static GridColumnSettings last_change => new GridColumnSettings(Columns.LastChange.ToString(), "Last Change", width: DIYGrid.DefaultColumnWidths.Time, horizontal_alignment: HorizontalAlignment.Right); - public static GridColumnSettings user => new GridColumnSettings(Columns.User.ToString(), DIYGrid.DefaultColumnHeaders.User, width: DIYGrid.DefaultColumnWidths.User, horizontal_alignment: HorizontalAlignment.Right); - public static GridColumnSettings account => new GridColumnSettings(Columns.Account.ToString(), DIYGrid.DefaultColumnHeaders.Account, width: DIYGrid.DefaultColumnWidths.Account, horizontal_alignment: HorizontalAlignment.Right); - public static GridColumnSettings exchange => new GridColumnSettings(Columns.Exchange.ToString(), DIYGrid.DefaultColumnHeaders.Exchange, width: DIYGrid.DefaultColumnWidths.Exchange, horizontal_alignment: HorizontalAlignment.Right); + public static GridColumnSettings symbol => new GridColumnSettings(Columns.Symbol.ToString(), DIYGrid.DefaultColumnHeaders.Symbol, width: DIYGrid.DefaultColumnWidths.Symbol, horizontal_alignment: HorizontalAlignment.Left); + public static GridColumnSettings time_created => new GridColumnSettings(Columns.Time.ToString(), "Time", width: DIYGrid.DefaultColumnWidths.Time, horizontal_alignment: HorizontalAlignment.Left); + public static GridColumnSettings last_change => new GridColumnSettings(Columns.LastChange.ToString(), "Last Change", width: DIYGrid.DefaultColumnWidths.Time, horizontal_alignment: HorizontalAlignment.Left); + public static GridColumnSettings user => new GridColumnSettings(Columns.User.ToString(), DIYGrid.DefaultColumnHeaders.User, width: DIYGrid.DefaultColumnWidths.User, horizontal_alignment: HorizontalAlignment.Left); + public static GridColumnSettings account => new GridColumnSettings(Columns.Account.ToString(), DIYGrid.DefaultColumnHeaders.Account, width: DIYGrid.DefaultColumnWidths.Account, horizontal_alignment: HorizontalAlignment.Left); + public static GridColumnSettings exchange => new GridColumnSettings(Columns.Exchange.ToString(), DIYGrid.DefaultColumnHeaders.Exchange, width: DIYGrid.DefaultColumnWidths.Exchange, horizontal_alignment: HorizontalAlignment.Left); public static GridColumnSettings time_in_force => new GridColumnSettings(Columns.TimeInForce.ToString(), DIYGrid.DefaultColumnHeaders.TimeInForce, width: DIYGrid.DefaultColumnWidths.TimeInForce); public static GridColumnSettings order_type => new GridColumnSettings(Columns.OrderType.ToString(), DIYGrid.DefaultColumnHeaders.OrderType, width: DIYGrid.DefaultColumnWidths.OrderType); public static GridColumnSettings root_algo_type => new GridColumnSettings(Columns.RootAlgoType.ToString(), DIYGrid.DefaultColumnHeaders.RootAlgoType, width: DIYGrid.DefaultColumnWidths.OrderType); - public static GridColumnSettings sub_type => new GridColumnSettings(Columns.SubType.ToString(), "Sub Type", width: 80, horizontal_alignment: HorizontalAlignment.Right); - public static GridColumnSettings side => new GridColumnSettings(Columns.Side.ToString(), "B/S", width: DIYGrid.DefaultColumnWidths.Side, horizontal_alignment: HorizontalAlignment.Right); + public static GridColumnSettings sub_type => new GridColumnSettings(Columns.SubType.ToString(), "Sub Type", width: 80, horizontal_alignment: HorizontalAlignment.Left); + public static GridColumnSettings side => new GridColumnSettings(Columns.Side.ToString(), "B/S", width: DIYGrid.DefaultColumnWidths.Side, horizontal_alignment: HorizontalAlignment.Left); public static GridColumnSettings quantity => new GridColumnSettings(Columns.Size.ToString(), "Qty", width: DIYGrid.DefaultColumnWidths.Size, horizontal_alignment: HorizontalAlignment.Right); public static GridColumnSettings price => new GridColumnSettings(Columns.Price.ToString(), DIYGrid.DefaultColumnHeaders.Price, width: DIYGrid.DefaultColumnWidths.Price, horizontal_alignment: HorizontalAlignment.Right); public static GridColumnSettings filled_quantity => new GridColumnSettings(Columns.Filled.ToString(), "Filled Qty", width: DIYGrid.DefaultColumnWidths.Filled, horizontal_alignment: HorizontalAlignment.Right); public static GridColumnSettings avg_price => new GridColumnSettings(Columns.AvgFilled.ToString(), "Avg Filled", width: DIYGrid.DefaultColumnWidths.AveragePrice, horizontal_alignment: HorizontalAlignment.Right); public static GridColumnSettings last_price => new GridColumnSettings(Columns.LastPrice.ToString(), "Last Filled", width: DIYGrid.DefaultColumnWidths.LastPrice, horizontal_alignment: HorizontalAlignment.Right); - public static GridColumnSettings status => new GridColumnSettings(Columns.Status.ToString(), "Status", width: 80, horizontal_alignment: HorizontalAlignment.Right); - public static GridColumnSettings last_message => new GridColumnSettings(Columns.LastMessage.ToString(), "Last Message", width: DIYGrid.DefaultColumnWidths.LastMessage, horizontal_alignment: HorizontalAlignment.Right); - public static GridColumnSettings security_key => new GridColumnSettings(Columns.SecurityKey.ToString(), DIYGrid.DefaultColumnHeaders.SecurityKey, is_visible: false, width: DIYGrid.DefaultColumnWidths.SecurityKey, horizontal_alignment: HorizontalAlignment.Right); - public static GridColumnSettings order_key => new GridColumnSettings(Columns.OrderKey.ToString(), DIYGrid.DefaultColumnHeaders.OrderKey, width: DIYGrid.DefaultColumnWidths.OrderKey, horizontal_alignment: HorizontalAlignment.Right); - public static GridColumnSettings parent_key => new GridColumnSettings(Columns.ParentKey.ToString(), "Parent Key", width: DIYGrid.DefaultColumnWidths.ParentKey, horizontal_alignment: HorizontalAlignment.Right); - public static GridColumnSettings attached_target_order_key => new GridColumnSettings(Columns.AttachedTargetOrderKey.ToString(), DIYGrid.DefaultColumnHeaders.AttachedTargetOrderKey, width: DIYGrid.DefaultColumnWidths.ParentKey, horizontal_alignment: HorizontalAlignment.Right); + public static GridColumnSettings status => new GridColumnSettings(Columns.Status.ToString(), "Status", width: 80, horizontal_alignment: HorizontalAlignment.Left); + public static GridColumnSettings last_message => new GridColumnSettings(Columns.LastMessage.ToString(), "Last Message", width: DIYGrid.DefaultColumnWidths.LastMessage, horizontal_alignment: HorizontalAlignment.Left); + public static GridColumnSettings security_key => new GridColumnSettings(Columns.SecurityKey.ToString(), DIYGrid.DefaultColumnHeaders.SecurityKey, is_visible: false, width: DIYGrid.DefaultColumnWidths.SecurityKey, horizontal_alignment: HorizontalAlignment.Left); + public static GridColumnSettings order_key => new GridColumnSettings(Columns.OrderKey.ToString(), DIYGrid.DefaultColumnHeaders.OrderKey, width: DIYGrid.DefaultColumnWidths.OrderKey, horizontal_alignment: HorizontalAlignment.Left); + public static GridColumnSettings parent_key => new GridColumnSettings(Columns.ParentKey.ToString(), "Parent Key", width: DIYGrid.DefaultColumnWidths.ParentKey, horizontal_alignment: HorizontalAlignment.Left); + public static GridColumnSettings attached_target_order_key => new GridColumnSettings(Columns.AttachedTargetOrderKey.ToString(), DIYGrid.DefaultColumnHeaders.AttachedTargetOrderKey, width: DIYGrid.DefaultColumnWidths.ParentKey, horizontal_alignment: HorizontalAlignment.Left); public static IEnumerable sorting_criterion { get { From b0759eba62c272050fa1767ef64c11550c98ad2a Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 9 May 2019 10:58:21 -0500 Subject: [PATCH 43/85] OrderBookRowVM - fix root badge color --- .../Trading/OrderBook/OrderBookRowVM.cs | 22 ++++++++++++++----- src/client/spark/Orders/OrderUtils.cs | 5 ++++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs index eceb5a03b..355a5e3bb 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs @@ -18,6 +18,7 @@ namespace mbts.spark { const string _unknown_security_price_format_string = "0.#########"; const int _unknown_security_price_num_fractional_digits = 9; readonly OrderBookVM _owner; + Color? _root_algo_badge_color_backing; Brush _badge_fill, _root_algo_badge_fill; IRaiseCanExecuteChangedCommand _show_order_modify_cmd, _toggle_expanded_cmd, _cancel_cmd, _detach_command, _attach_command, _update_command, _modify_order_quantity_andor_price, _revert_prospective_order_quantity_andor_price; bool _is_parent, _is_expanded, _is_selected, _is_attachable, _is_detachable; @@ -106,7 +107,7 @@ namespace mbts.spark { // Root Algo public CustomOrderDefinition.CustomOneofCase root_algo_order_type => order_tuple.root_algo.value.CustomCase; public OrderMetadata root_algo_order_metadata => _owner.get_metadata(root_algo_order_type); - public Brush root_algo_badge_fill => _root_algo_badge_fill ?? (_root_algo_badge_fill = order_tuple.root_badge_fill(root_algo_order_metadata)); + public Brush root_algo_badge_fill => _root_algo_badge_fill ?? (_root_algo_badge_fill = new SolidColorBrush(_root_algo_badge_color)); public string root_algo_display_name => order_tuple.root_algo.name; // Tree @@ -199,6 +200,18 @@ namespace mbts.spark { _select_me(); }) ); + + // private props + Color _root_algo_badge_color { + get => _root_algo_badge_color_backing ?? (_root_algo_badge_color_backing = order_tuple.root_badge_color(root_algo_order_metadata)).Value; + set { + if (_root_algo_badge_color_backing != value) { + _root_algo_badge_color_backing = value; + _root_algo_badge_fill = null; + } + } + } + #region IHierarchicalRow bool IHierarchicalRow.is_parent => is_parent; bool IHierarchicalRow.is_expanded => is_expanded; @@ -269,10 +282,9 @@ namespace mbts.spark { if (old_order_tuple.display_name() != order_tuple.display_name()) RaisePropertyChanged(nameof(display_name)); // root badge fill - if (old_order_tuple.root_meta?.badge_color != order_tuple.root_meta?.badge_color) { - // invalidate by setting to null (will be re-instantiated in the public getter) - _root_algo_badge_fill = null; - RaisePropertyChanged(nameof(root_algo_badge_fill)); + if (_root_algo_badge_color_backing.HasValue) { + // only need to invalidate it if it had a value (it is lazily instantiated) + _root_algo_badge_color = order_tuple.root_badge_color(root_algo_order_metadata); } // root display name if (old_order_tuple.root_algo.name != order_tuple.root_algo.name) diff --git a/src/client/spark/Orders/OrderUtils.cs b/src/client/spark/Orders/OrderUtils.cs index 3eb377cc1..ebab05067 100644 --- a/src/client/spark/Orders/OrderUtils.cs +++ b/src/client/spark/Orders/OrderUtils.cs @@ -251,8 +251,11 @@ namespace mbts.spark { } } + public static Color root_badge_color(this OrderSecurityAlgoMeta order_sam, OrderMetadata root_order_meta) => + order_sam.root_meta?.badge_color is Color c ? c : root_order_meta.Color.rgb_color(); + public static Brush root_badge_fill(this OrderSecurityAlgoMeta order_sam, OrderMetadata root_order_meta) => - order_sam.root_meta?.badge_color is Color c ? c.to_brush() : root_order_meta.Color.rgb_brush(); + new SolidColorBrush(root_badge_color(order_sam, root_order_meta)); public static string order_collection_summary(double best_price, int best_price_volume, int total_volume, ContractSpecificationImmutable spec) { if (total_volume == 0) From b1498f3df1507f8718a37cc63bf786e23652cf15 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 9 May 2019 11:03:08 -0500 Subject: [PATCH 44/85] OrderBookRowVM - forgot to raise property changed when root badge color changes --- src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs index 355a5e3bb..a7cfa95a1 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs @@ -285,6 +285,7 @@ namespace mbts.spark { if (_root_algo_badge_color_backing.HasValue) { // only need to invalidate it if it had a value (it is lazily instantiated) _root_algo_badge_color = order_tuple.root_badge_color(root_algo_order_metadata); + RaisePropertyChanged(nameof(root_algo_badge_fill)); } // root display name if (old_order_tuple.root_algo.name != order_tuple.root_algo.name) From 004a7b13959440da3f4e3e3e4a8b8df3a1de5f94 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 9 May 2019 11:17:03 -0500 Subject: [PATCH 45/85] OrderBookRowVM - don't hook up to invalidated event until '_modify_order_quantity_andor_price' cmd is created --- .../Trading/OrderBook/OrderBookRowVM.cs | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs index a7cfa95a1..d16edf8fb 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs @@ -38,11 +38,6 @@ namespace mbts.spark { _is_expanded = is_expanded; prospective_quantity = this.quantity; prospective_price = this.price; - this.invalidated += OrderBookRowVM_invalidated; - } - - private void OrderBookRowVM_invalidated() { - modify_order_quantity_andor_price?.TryRaiseCanExecuteChanged(); } //// This takes care of the case where the OB has all rows visible @@ -155,8 +150,28 @@ namespace mbts.spark { public ICommand cancel_command => _cancel_cmd ?? (_cancel_cmd = new DelegateCommand(() => _owner.cancel_order(this), () => status == OrderStatus.Working || status == OrderStatus.Disconnected)); - public ICommand modify_order_quantity_andor_price => _modify_order_quantity_andor_price ?? - (_modify_order_quantity_andor_price = new DelegateCommand( + public ICommand modify_order_quantity_andor_price { + get { + if (_modify_order_quantity_andor_price == null) { + _modify_order_quantity_andor_price = new DelegateCommand(execute, can_execute); + invalidated += _modify_order_quantity_andor_price.RaiseCanExecuteChanged; + // impl + void execute() { + if (is_qty_dirty && is_price_dirty) { + _owner.modify_order_price_and_quantity(this, prospective_price, prospective_quantity); + } else if (is_qty_dirty) { + _owner.modify_order_quantity(this, prospective_quantity); + } else if (is_price_dirty) { + _owner.modify_order_price(this, this.prospective_price); + } + _unselect_me(); + } + bool can_execute() => status == OrderStatus.Working && (is_price_dirty || is_qty_dirty); + } + return _modify_order_quantity_andor_price; + } + } + () => { if (is_qty_dirty && is_price_dirty) { _owner.modify_order_price_and_quantity(this, prospective_price, prospective_quantity); From c7e3bdcd6cc1ad2e3897d979507a9d1ca384f70d Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 9 May 2019 11:18:27 -0500 Subject: [PATCH 46/85] OrderBookRowVM - normalize spacing --- .../Trading/OrderBook/OrderBookRowVM.cs | 28 ++++--------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs index d16edf8fb..650e5c0d2 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs @@ -24,13 +24,10 @@ namespace mbts.spark { bool _is_parent, _is_expanded, _is_selected, _is_attachable, _is_detachable; int _depth_level; SecurityImmutable? _security => order_tuple.security; - - // for mini-ob uint _prospective_quantity; double _prospective_price; - public OrderBookRowVM(OrderBookVM owner, OrderSecurityAlgoMeta order_tuple, bool is_expanded) { _owner = owner; this.order_tuple = order_tuple; @@ -172,30 +169,16 @@ namespace mbts.spark { } } + public ICommand revert_prospective_order_quantity_andor_price => _revert_prospective_order_quantity_andor_price ?? + (_revert_prospective_order_quantity_andor_price = new DelegateCommand( () => { - if (is_qty_dirty && is_price_dirty) { - _owner.modify_order_price_and_quantity(this, prospective_price, prospective_quantity); - } else if (is_qty_dirty) { - _owner.modify_order_quantity(this, prospective_quantity); - } else if (is_price_dirty) { - _owner.modify_order_price(this, this.prospective_price); - } - _unselect_me(); - }, + prospective_price = price; + prospective_quantity = quantity; + }, () => this.status == OrderStatus.Working && (is_price_dirty || is_qty_dirty) ) ); - public ICommand revert_prospective_order_quantity_andor_price => _revert_prospective_order_quantity_andor_price ?? - (_revert_prospective_order_quantity_andor_price = new DelegateCommand( - () => { - prospective_price = price; - prospective_quantity = quantity; - }, - () => this.status == OrderStatus.Working && (is_price_dirty || is_qty_dirty) - ) -); - public ICommand detach_command => _detach_command ?? (_detach_command = new DelegateCommand(() => { _owner.detach_orders(this); @@ -331,7 +314,6 @@ namespace mbts.spark { // this will update the cancel column in the mini ob cancel_command.TryRaiseCanExecuteChanged(); } - // always refresh attach and detach b/c // conditionally raise invalidated if ((dirty & ViewRefresh.invalidate_mask) != 0) From 1f94454d635b358ad1528230e540d39c34d3eceb Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 9 May 2019 11:20:10 -0500 Subject: [PATCH 47/85] SPARK-2421 Default ladder to new fancy param grid --- src/lib/config/protos/UI.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/config/protos/UI.proto b/src/lib/config/protos/UI.proto index 71b9702c8..467b433ec 100644 --- a/src/lib/config/protos/UI.proto +++ b/src/lib/config/protos/UI.proto @@ -68,7 +68,7 @@ message LadderKnobs { optional uint32 max_open_ladders = 4 [default = 75]; optional uint32 security_timeout_secs = 5 [default = 60]; optional bool open_algo_popup_on_select = 6 [default = false]; - optional bool use_algo_param_grid = 7 [default = false]; + optional bool use_algo_param_grid = 7 [default = true]; } message ShellPingOptions { From 793eb14ed75ad694e3750b09f157c52f9c42de72 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 9 May 2019 11:39:55 -0500 Subject: [PATCH 48/85] AlgoDefinitionMetadata - null check _favorite_templates in GetHashCode --- .../Trading/AlgoStudio/Model/Metadata/AlgoDefinitionMetadata.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/Modules/Trading/AlgoStudio/Model/Metadata/AlgoDefinitionMetadata.cs b/src/client/spark/Modules/Trading/AlgoStudio/Model/Metadata/AlgoDefinitionMetadata.cs index 940b0aa92..aa536d1bf 100644 --- a/src/client/spark/Modules/Trading/AlgoStudio/Model/Metadata/AlgoDefinitionMetadata.cs +++ b/src/client/spark/Modules/Trading/AlgoStudio/Model/Metadata/AlgoDefinitionMetadata.cs @@ -70,7 +70,7 @@ namespace mbts.spark { is_hidden.GetHashCode(), abbreviation == null ? 0 : abbreviation.GetHashCode(), badge_color?.GetHashCode() ?? 0, - EqualityComparer.aggregate_hash_limited(_favorite_templates)); + _favorite_templates == null ? 0 : EqualityComparer.aggregate_hash_limited(_favorite_templates)); public static bool operator ==(AlgoDefinitionMetadata a, AlgoDefinitionMetadata b) => a.Equals(b); public static bool operator !=(AlgoDefinitionMetadata a, AlgoDefinitionMetadata b) => !a.Equals(b); From f16ed40c821ff09bd8fc3cd1aae736c68e439b03 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 9 May 2019 12:12:21 -0500 Subject: [PATCH 49/85] OrderUtils.badge_fill does not need the OrderRecordImmutable arg --- .../spark/Modules/Trading/Fills/FillsVM.cs | 2 +- .../Trading/OrderBook/OrderBookRowVM.cs | 6 +++--- src/client/spark/Orders/OrderUtils.cs | 18 ++++-------------- 3 files changed, 8 insertions(+), 18 deletions(-) diff --git a/src/client/spark/Modules/Trading/Fills/FillsVM.cs b/src/client/spark/Modules/Trading/Fills/FillsVM.cs index 841e32391..d3a47734d 100644 --- a/src/client/spark/Modules/Trading/Fills/FillsVM.cs +++ b/src/client/spark/Modules/Trading/Fills/FillsVM.cs @@ -233,7 +233,7 @@ namespace mbts.spark { public OrderType order_type => _order.order_type(); public string display_name => OrderUtils.display_name(_order, _fill_tuple.algo); public OrderMetadata order_metadata => _order_metadata ?? (_order_metadata = _model.factory.get_metadata(_order.Def.CustomDef.CustomCase)); - public Brush badge_fill => _badge_fill ?? (_badge_fill = OrderUtils.badge_fill(_order, _fill_tuple.algo_meta, order_metadata)); + public Brush badge_fill => _badge_fill ?? (_badge_fill = OrderUtils.badge_fill(_fill_tuple.algo_meta, order_metadata)); public string account { get; private set; } public string user => _order.user(); public Venue exchange => security.Exchange; diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs index 650e5c0d2..a93c6bbc3 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs @@ -58,7 +58,7 @@ namespace mbts.spark { public OrderSecurityAlgoMeta order_tuple { get; private set; } public OrderRecordImmutable order => order_tuple.order; public OrderMetadata order_metadata => _owner.get_metadata(this); - public Brush badge_fill => _badge_fill ?? (_badge_fill = order_tuple.badge_fill(order_metadata)); + public Brush badge_fill => _badge_fill ?? (_badge_fill = OrderUtils.badge_fill(order_tuple.meta, root_algo_order_metadata)); public uint order_key => order.Key; public uint parent_order_key => order.ParentKey; public uint security_key => order.security_key(); @@ -201,7 +201,7 @@ namespace mbts.spark { // private props Color _root_algo_badge_color { - get => _root_algo_badge_color_backing ?? (_root_algo_badge_color_backing = order_tuple.root_badge_color(root_algo_order_metadata)).Value; + get => _root_algo_badge_color_backing ?? (_root_algo_badge_color_backing = OrderUtils.badge_color(order_tuple.root_meta, root_algo_order_metadata)).Value; set { if (_root_algo_badge_color_backing != value) { _root_algo_badge_color_backing = value; @@ -282,7 +282,7 @@ namespace mbts.spark { // root badge fill if (_root_algo_badge_color_backing.HasValue) { // only need to invalidate it if it had a value (it is lazily instantiated) - _root_algo_badge_color = order_tuple.root_badge_color(root_algo_order_metadata); + _root_algo_badge_color = OrderUtils.badge_color(order_tuple.root_meta, root_algo_order_metadata); RaisePropertyChanged(nameof(root_algo_badge_fill)); } // root display name diff --git a/src/client/spark/Orders/OrderUtils.cs b/src/client/spark/Orders/OrderUtils.cs index ebab05067..d8096a508 100644 --- a/src/client/spark/Orders/OrderUtils.cs +++ b/src/client/spark/Orders/OrderUtils.cs @@ -240,22 +240,12 @@ namespace mbts.spark { } return false; } - public static Brush badge_fill(this OrderSecurityAlgoMeta order_sam, OrderMetadata order_meta) => - badge_fill(order_sam.order, order_sam.meta, order_meta); - public static Brush badge_fill(OrderRecordImmutable order, AlgoDefinitionMetadata? algo_meta, OrderMetadata order_meta) { - if (algo_meta?.badge_color is Color c) { - return c.to_brush(); - } else { - return order_meta.Color.rgb_brush(); - } - } + public static Brush badge_fill(AlgoDefinitionMetadata? algo_meta, OrderMetadata order_meta) => + new SolidColorBrush(badge_color(algo_meta, order_meta)); - public static Color root_badge_color(this OrderSecurityAlgoMeta order_sam, OrderMetadata root_order_meta) => - order_sam.root_meta?.badge_color is Color c ? c : root_order_meta.Color.rgb_color(); - - public static Brush root_badge_fill(this OrderSecurityAlgoMeta order_sam, OrderMetadata root_order_meta) => - new SolidColorBrush(root_badge_color(order_sam, root_order_meta)); + public static Color badge_color(AlgoDefinitionMetadata? algo_meta, OrderMetadata order_meta) => + algo_meta?.badge_color ?? order_meta.Color.rgb_color(); public static string order_collection_summary(double best_price, int best_price_volume, int total_volume, ContractSpecificationImmutable spec) { if (total_volume == 0) From 526ec306aea754421406c60cb565ab79a1dafe7f Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 9 May 2019 12:41:35 -0500 Subject: [PATCH 50/85] (Root)OrderType col filters ItemTemplate uses 'OrderTypeItem' (displays badge and name) --- .../Trading/AlgoStudio/Model/AlgoData.cs | 19 ++++++++++---- src/client/spark/Orders/OrdersModel.cs | 26 +++++++++++-------- .../Filters/Views/FilterTemplates.xaml | 5 ++++ 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/client/spark/Modules/Trading/AlgoStudio/Model/AlgoData.cs b/src/client/spark/Modules/Trading/AlgoStudio/Model/AlgoData.cs index 2ad70d0b7..3acd47fcc 100644 --- a/src/client/spark/Modules/Trading/AlgoStudio/Model/AlgoData.cs +++ b/src/client/spark/Modules/Trading/AlgoStudio/Model/AlgoData.cs @@ -2,6 +2,7 @@ using bts.utils; using mbts.spark.utilities; using System; +using System.Windows.Media; namespace mbts.spark { struct AlgoData : IEquatable { @@ -39,20 +40,28 @@ namespace mbts.spark { struct AlgoDataVM : IEquatable { public readonly AlgoData data; + public readonly AlgoDefinitionMetadata? meta; - public AlgoDataVM(AlgoData data) => + public AlgoDataVM(AlgoData data, AlgoDefinitionMetadata? meta = null) { this.data = data; + this.meta = meta; + } + public string name => data.name; + public Brush badge_fill => OrderUtils.badge_fill(meta, OrderDataHelper.get_metadata(data.value.CustomCase)); public AlgoDisplayGroup group => data.key.display_group(); public override string ToString() => data.name; - public static AlgoDataVM from_data(AlgoData data) => - new AlgoDataVM(data); + public static AlgoDataVM from_data(AlgoData data, AlgoDefinitionMetadata? meta) => + new AlgoDataVM(data, meta); - public bool Equals(AlgoDataVM other) => data == other.data; + public static AlgoDataVM from_data((AlgoData algo, AlgoDefinitionMetadata? meta) tup) => + new AlgoDataVM(tup.algo, tup.meta); + + public bool Equals(AlgoDataVM other) => data == other.data && meta == other.meta; public override bool Equals(object obj) => obj is AlgoDataVM other && Equals(other); - public override int GetHashCode() => data.GetHashCode(); + public override int GetHashCode() => EqualityComparer.aggregate_hash(data.GetHashCode(), meta.GetHashCode()); } } \ No newline at end of file diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 251f1966b..078da0e39 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -84,15 +84,16 @@ namespace mbts.spark { _security_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); _security_working_orders_with_piq_demuxer = new TableUpdateDemuxer(o => o.order.Key, o => o.order.security_key()); _security_positions_demuxer = new TableUpdateDemuxer(o => o.RecordId, p => p.SecurityId); - var algo_meta_join = algo_model.definitions.LeftJoin( + studio_algos_with_meta = algo_model.definitions.LeftJoin( algo_model.metadata_service.subscribe_metadata().Table(m => m.id), d => d.key, m => m.id, (algo, k, meta) => (algo, meta), (algo, k) => (algo, null), - t => t.algo.key); + t => t.algo.key) + .Cache(); var order_secs_algo_meta_join = on_orders.LeftJoin( - algo_meta_join, + studio_algos_with_meta, t => t.order.Key, am => am.algo.key, t => new AlgoKey(new bts.persist.algo.AlgoPersistedKey(t.order.algo_template_key())), @@ -178,6 +179,7 @@ namespace mbts.spark { public IObservable> on_fill_tuples { get; } public IObservable> working_orders { get; } public IObservable> on_positions { get; } + public IObservable> studio_algos_with_meta { get; } public AdminModel admin_model { get; } public IAlgoStudioModel algo_model { get; } @@ -436,16 +438,18 @@ namespace mbts.spark { return rod; }).Select(r => r.last_update); - public static IObservable> subscribe_all_algos(this OrdersModel orders_model) => - orders_model.algo_model.definitions.MergeTable(Observable.Return( - new TableUpdate( - ImmutableDictionary.Empty.AddRange(orders_model.factory.default_algos(OrderDataHelper.all_order_types)), - Array.Empty(), - Array.Empty>(), - Array.Empty(), - d => d.key + public static IObservable> subscribe_all_algos(this OrdersModel orders_model) => + orders_model.studio_algos_with_meta.MergeTable(Observable.Return( + new TableUpdate<(AlgoData algo, AlgoDefinitionMetadata? meta), AlgoKey>( + ImmutableDictionary.Empty.AddRange(orders_model.factory.default_algos(OrderDataHelper.all_order_types).Select(kv => new KeyValuePair(kv.Key, (kv.Value, null)))), + Array.Empty<(AlgoData,AlgoDefinitionMetadata?)>(), + Array.Empty>(), + Array.Empty<(AlgoData,AlgoDefinitionMetadata?)>(), + d => d.algo.key ).as_snapshot())); + //public static IObservable> subscribe_all_algos_with_meta(this OrdersModel orders_model) => + public static async Task order_from_key(this OrdersModel orders_model, uint order_key) { (var tu, var success) = await orders_model.on_orders.FirstAsyncTask(t => t.state.ContainsKey(order_key), TimeSpan.FromSeconds(3)); if (success && tu.state.TryGetValue(order_key, out var rec)) { diff --git a/src/client/spark/view/Controls/DataGridEx/Filters/Views/FilterTemplates.xaml b/src/client/spark/view/Controls/DataGridEx/Filters/Views/FilterTemplates.xaml index 7b6e6fb47..b21f8c206 100644 --- a/src/client/spark/view/Controls/DataGridEx/Filters/Views/FilterTemplates.xaml +++ b/src/client/spark/view/Controls/DataGridEx/Filters/Views/FilterTemplates.xaml @@ -163,6 +163,11 @@ + + + + + From 04a68a4e96b28c1bbcff13b1a8a5bb57f6940122 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 9 May 2019 13:43:47 -0500 Subject: [PATCH 51/85] OrderBookRowVM - fix typeo in badge_color prop (was using 'root') --- src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs index a93c6bbc3..97314434b 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs @@ -58,7 +58,7 @@ namespace mbts.spark { public OrderSecurityAlgoMeta order_tuple { get; private set; } public OrderRecordImmutable order => order_tuple.order; public OrderMetadata order_metadata => _owner.get_metadata(this); - public Brush badge_fill => _badge_fill ?? (_badge_fill = OrderUtils.badge_fill(order_tuple.meta, root_algo_order_metadata)); + public Brush badge_fill => _badge_fill ?? (_badge_fill = OrderUtils.badge_fill(order_tuple.meta, order_metadata)); public uint order_key => order.Key; public uint parent_order_key => order.ParentKey; public uint security_key => order.security_key(); From e14a5b2575c0be66c5e58f22070973ee7774d2ad Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Thu, 9 May 2019 18:41:21 +0000 Subject: [PATCH 52/85] update SDK readme --- src/sdk/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/README.md b/src/sdk/README.md index ee11093a2..63a91c61b 100644 --- a/src/sdk/README.md +++ b/src/sdk/README.md @@ -67,7 +67,7 @@ $ cd spark_sdk ### Development -Algo Orders typically implement the IAlgoOrder interface to respond to events and use the IAlgoServices interface to create and control child orders as well as manage their own life cycles. See [examples/order_plugin_example.cpp](examples/order_plugin_example.cpp) for a fully fledged example +Algo Orders typically implement the [IAlgoOrder](include/spark/lib/orderman/orders/algo/IAlgoOrder.h) interface to respond to events and use the [IAlgoServices](include/spark/lib/orderman/orders/algo/IAlgoServices.h) interface to create and control child orders as well as manage their own life cycles. See [examples/order_plugin_example.cpp](examples/order_plugin_example.cpp) for a fully fledged example Look at [examples/md_publisher_example.cpp](examples/md_publisher_example.cpp) to look at how to build a custom market data producer From 3a6f9a608d419ff7d41e77edc099945063ad76a5 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 9 May 2019 14:26:42 -0500 Subject: [PATCH 53/85] OrderBookVM.send_attached_orders_async uses CustomDef.actual() to unwrap composite algos --- src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs index 2da113487..03d0e295e 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs @@ -248,10 +248,7 @@ namespace mbts.spark { var order_tups = selected_rows.Select(r => r.order_tuple).ToList(); var mod_algo_vm = attach_vm.algo_vm.algo_root; await Task.WhenAll(order_tups.Select(t => { - var algo = t.order.Def.CustomDef.CloneToMutable(); - if (algo.CustomCase == CustomOrderDefinition.CustomOneofCase.Composite) { - algo = algo.Composite.Definition; - } + var algo = t.order.Def.CustomDef.actual().CloneToMutable(); mod_algo_vm.assign_to(algo); if (algo_template_key != 0) { algo = new CustomOrderDefinition { From 38f42fe747138b78e0a268649c121a82f67c6e55 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 9 May 2019 14:28:26 -0500 Subject: [PATCH 54/85] SPARK-2417 Attach window price, qty --- src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs index 03d0e295e..4a0fcb5e1 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs @@ -277,7 +277,7 @@ namespace mbts.spark { connect_common_selected_params(attach_vm), _selected_params_mon.subscribe_algos().Subscribe(algos => { bool show_price, show_size; - using (var en = algos.Select(a => a.CustomCase).Distinct().GetEnumerator()) { + using (var en = algos.Select(a => a.actual().CustomCase).Distinct().GetEnumerator()) { if (show_price = show_size = en.MoveNext()) { do { if (show_price) show_price = AttachConf.show_price(en.Current); From 13e5c017e2e667be3a68616f712e39ba7e966c63 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 9 May 2019 15:53:12 -0500 Subject: [PATCH 55/85] miniob - set orderbookvm.is_attaching_orders to false when attach popup closes --- .../spark/Modules/Trading/OrderBook/OrderBookColumns.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs index 32ef7d1a6..ab74a045d 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs @@ -1062,6 +1062,10 @@ namespace mbts.spark { popup.SetBinding(Popup.DataContextProperty, new Binding("PlacementTarget.DataContext") { RelativeSource = new RelativeSource(RelativeSourceMode.Self) }); + popup.Closed += (s, e) => { + if (popup.try_find_ancestor_of_type(out MiniOrderBookView mini_ob) && mini_ob.DataContext is OrderBookVM ob_vm) + ob_vm.is_attaching_orders = false; + }; var the_whole_thing = new Grid(); the_whole_thing.Children.Add(update_button); From 78373c61cee51004f78118aec62798c81e0e8f4b Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 9 May 2019 16:09:55 -0500 Subject: [PATCH 56/85] miniob - hide price/qty param rows in attach popup --- .../spark/Modules/Trading/AlgoStudio/VM/AlgoParamRowVM.cs | 4 ++++ .../View/AlgoEditorGridView/AlgoEditorGridView.xaml.cs | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoParamRowVM.cs b/src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoParamRowVM.cs index ae497709a..e7ebaa288 100644 --- a/src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoParamRowVM.cs +++ b/src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoParamRowVM.cs @@ -18,12 +18,16 @@ namespace mbts.spark { this.algo = algo; this.parent = parent; _template_key = template_key; + param.permissions_changed += on_permissions_changed; + _dispose = Disposable.Create(() => param.permissions_changed -= on_permissions_changed); + void on_permissions_changed(AlgoParamPermissions oldval, AlgoParamPermissions newval) => RaisePropertyChanged(nameof(permissions)); } public event Action invalidated; public IAlgoVM algo { get; set; } public IAlgoParamVM param { get; } + public AlgoParamPermissions permissions => param.permissions; public AlgoParamRowVM parent { get; } public int level => parent == null ? 0 : parent.level + 1; public string path_disambiguation { diff --git a/src/client/spark/Modules/Trading/AlgoStudio/View/AlgoEditorGridView/AlgoEditorGridView.xaml.cs b/src/client/spark/Modules/Trading/AlgoStudio/View/AlgoEditorGridView/AlgoEditorGridView.xaml.cs index f45e2849b..c8dbb61a0 100644 --- a/src/client/spark/Modules/Trading/AlgoStudio/View/AlgoEditorGridView/AlgoEditorGridView.xaml.cs +++ b/src/client/spark/Modules/Trading/AlgoStudio/View/AlgoEditorGridView/AlgoEditorGridView.xaml.cs @@ -52,6 +52,8 @@ namespace mbts.ui.spark { // AlgoParamRows AlgoParamRowsViewSource = new CollectionViewSource>(); BindingOperations.SetBinding(AlgoParamRowsViewSource, CollectionViewSource.SourceProperty, new Binding(nameof(AlgoParamRows)) { Source = this }); + AlgoParamRowsViewSource.IsLiveFilteringRequested = true; + AlgoParamRowsViewSource.LiveFilteringProperties.Add(nameof(AlgoParamRowVM.permissions)); AlgoParamRowsViewSource.Filter += param_rows_filter; DataGridEditHelper.set_hooks(this); void param_rows_filter(object sender, FilterEventArgs e) => From f8ddaab88464fac15fb145c537232da94ab3a22e Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 9 May 2019 18:00:12 -0500 Subject: [PATCH 57/85] OrderBook - (Root)OrderType col filters show green beaker when filter active on startup/load workspace --- .../Filters/VM/DiscreteValuesFilterVM.cs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/client/spark/view/Controls/DataGridEx/Filters/VM/DiscreteValuesFilterVM.cs b/src/client/spark/view/Controls/DataGridEx/Filters/VM/DiscreteValuesFilterVM.cs index 5d9eb34e4..fbc33e8e5 100644 --- a/src/client/spark/view/Controls/DataGridEx/Filters/VM/DiscreteValuesFilterVM.cs +++ b/src/client/spark/view/Controls/DataGridEx/Filters/VM/DiscreteValuesFilterVM.cs @@ -262,7 +262,7 @@ namespace mbts.spark.filters { readonly Func> _load_state; readonly Func, TState> _get_state; readonly IEqualityComparer _comparer; - readonly HashSet _selected_keys = new HashSet(); + readonly HashSet _selected_keys = new HashSet(), _desired_keys = new HashSet(); ListCollectionView _items_view; public TableUpdateItemsFilter(Func factory, Func row_key, Func key, Func> load_state, Func, TState> get_state) @@ -290,6 +290,12 @@ namespace mbts.spark.filters { _items.Replace(_key(ch.Old), ch.New); foreach (var v in table.added) _items.Add(v); + var newly_selected_keys = _desired_keys.Where(_selection_model.try_select).ToList(); + if (newly_selected_keys.Count > 0) { + _selected_keys.AddRange(newly_selected_keys); + _desired_keys.removeRange(newly_selected_keys); + _raise_filter_changed(); + } } public override TState get_state() => _get_state(_selection_model.selected_items.Select(_key)); @@ -309,8 +315,10 @@ namespace mbts.spark.filters { protected override bool _filter(TValue value) => _selected_keys.Contains(_key(value)); protected override void _load_state_internal(TState state) { - foreach (var k in _load_state(state).empty_if_null()) - _selection_model.try_select(k); + foreach (var k in _load_state(state).empty_if_null()) { + if (!_selection_model.try_select(k)) + _desired_keys.Add(k); + } } } #endregion @@ -337,4 +345,4 @@ namespace mbts.spark.filters { } } } -} \ No newline at end of file +} From 85f42d1e23ee59639e5c4d8be06e8c3cdceb13ac Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Fri, 10 May 2019 11:12:43 -0400 Subject: [PATCH 58/85] Join won't modify its child's price once it has joined --- .../orders/algo/algo_orders/JoinAlgo.cpp | 20 ++++++++++++++++++- .../orders/algo/algo_orders/JoinAlgo.h | 1 + 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/lib/orderman/orders/algo/algo_orders/JoinAlgo.cpp b/src/lib/orderman/orders/algo/algo_orders/JoinAlgo.cpp index 9c8d8e3be..e85a07f6e 100644 --- a/src/lib/orderman/orders/algo/algo_orders/JoinAlgo.cpp +++ b/src/lib/orderman/orders/algo/algo_orders/JoinAlgo.cpp @@ -3,6 +3,7 @@ #include "lib/orderman/order_factory/IOrderFactory.h" #include "lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.h" +#include "lib/os/os.h" #include "lib/utils/PBUtils.h" namespace bts { @@ -20,6 +21,7 @@ struct JoinAlgo::PImpl { ChildTemplate *child_template = nullptr; Book book_at_inception; Book last_book; + Price joined_price=Price(os::nan()); }; JoinAlgo::JoinAlgo() : me(new PImpl()) {} @@ -80,7 +82,8 @@ void JoinAlgo::do_update(api::orderman::Update const &update) { } Update child_update; - child_update.set_price(update.price()); + Price price_to_pass = get_price_to_pass(Price(update.price())); + child_update.set_price(price_to_pass.value()); child_update.set_quantity(update.quantity()); if (update.has_custom()) { update_join_threshold(); @@ -146,6 +149,7 @@ void JoinAlgo::try_to_join() { // Time to join Update child_update; + me->joined_price = Price(my_side_psz.price()); child_update.set_price(my_side_psz.price()); child_update.set_quantity(algo().quantity().value()); algo().update_child(me->child_key, child_update); @@ -161,6 +165,20 @@ void JoinAlgo::try_to_join() { } } +Price JoinAlgo::get_price_to_pass (Price price) const{ + Price rv; + switch (me->status) { + case Status::WAITING_FOR_MD: + case Status::WAITING_TO_JOIN: + rv = price; + break; + case Status::JOINED: + rv = me->joined_price; + break; + } + return rv; +} + void JoinAlgo::update_join_threshold() { me->min_quantity_to_join = SpreaderUtils::_quantity_threshold( algo().definition().custom_def().join().join_threshold(), algo().quantity()); diff --git a/src/lib/orderman/orders/algo/algo_orders/JoinAlgo.h b/src/lib/orderman/orders/algo/algo_orders/JoinAlgo.h index 114a9131b..e77aa382b 100644 --- a/src/lib/orderman/orders/algo/algo_orders/JoinAlgo.h +++ b/src/lib/orderman/orders/algo/algo_orders/JoinAlgo.h @@ -31,6 +31,7 @@ private: void update_join_threshold(); bool has_running_child() const; void try_to_join(); + Price get_price_to_pass (Price price) const; }; }} From 6f7851b7ba840956041f051fa9f1b116ac6c190f Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Fri, 10 May 2019 13:41:05 -0400 Subject: [PATCH 59/85] SPARK-2064: Simple sanity test --- src/client/spark_test/System/SpreaderTests.cs | 103 ++++++++++++++++-- .../manual/spreader_misc/simple_sanity.txt | 6 +- 2 files changed, 95 insertions(+), 14 deletions(-) diff --git a/src/client/spark_test/System/SpreaderTests.cs b/src/client/spark_test/System/SpreaderTests.cs index 90fa8123f..a752f6cae 100644 --- a/src/client/spark_test/System/SpreaderTests.cs +++ b/src/client/spark_test/System/SpreaderTests.cs @@ -72,27 +72,108 @@ namespace bts.tests test_account, test_user ); - var order1 = await OrderUtilities.send_order(_order_service, order1_def, -0.30, 1); + var spreader_order1_response = await OrderUtilities.send_order(_order_service, order1_def, -0.30, 1); - var spreader_orders = await OrderUtilities.wait_for_filtered_orders(_order_service, - o => o.Where(ordi => ordi.Key == order1.value.OrderKey).Count() > 0, - ord => ord.Where(ordi => ordi.Key == order1.value.OrderKey)); - Assert.IsTrue(spreader_orders.result.Count() > 0, "Can't find spreader order"); + var spreader_orders1 = await OrderUtilities.wait_for_filtered_orders(_order_service, + o => o.Where(ordi => ordi.Key == spreader_order1_response.value.OrderKey).Count() > 0, + ord => ord.Where(ordi => ordi.Key == spreader_order1_response.value.OrderKey)); + Assert.IsTrue(spreader_orders1.result.Count() > 0, "Can't find spreader order"); - var order = spreader_orders.result.Where(ordi => ordi.Key == order1.value.OrderKey).TryFirst(); - Assert.IsTrue(order.HasValue, "order not found"); - Assert.IsTrue(order.Value.State.Status == OrderStatus.Working, "Order is not Working"); + var working_spreader_order1 = spreader_orders1.result.Where(ordi => ordi.Key == spreader_order1_response.value.OrderKey).TryFirst(); + Assert.IsTrue(working_spreader_order1.HasValue, "Spreader order not found"); + Assert.IsTrue(working_spreader_order1.Value.State.Status == OrderStatus.Working, "Spreader order is not Working"); - //Verify that a quoting order of 18.60 shows up in the second leg. var quoting_orders = await OrderUtilities.wait_for_filtered_orders(_order_service, - o => o.Where(ordi => ordi.ParentKey == order1.value.OrderKey).Count() > 0, - ord => ord.Where(ordi => ordi.ParentKey == order1.value.OrderKey)); + o => o.Where(ordi => ordi.ParentKey == spreader_order1_response.value.OrderKey).Count() > 0, + ord => ord.Where(ordi => ordi.ParentKey == spreader_order1_response.value.OrderKey)); var quoting_order = quoting_orders.result.TryFirst(); Assert.IsTrue(quoting_order.HasValue, "order not found"); Assert.IsTrue(quoting_order.Value.State.Status == OrderStatus.Working, "Order is not Working"); Assert.IsTrue(quoting_order.Value.State.Price == 18.6, "Quoting order has the wrong price"); + var spreader_order1_cancel_ret = await OrderUtilities.cancel_orders(_order_service, new List {working_spreader_order1.Value}); + + var canceled_quoting_orders = await OrderUtilities.wait_for_filtered_orders(_order_service, + o => o.Where(ordi => ordi.ParentKey == spreader_order1_response.value.OrderKey && ordi.State.Status == OrderStatus.Canceled).Count() > 0, + ord => ord.Where(ordi => ordi.ParentKey == spreader_order1_response.value.OrderKey), + TimeSpan.FromSeconds(30)); + + var canceled_quoting_order = canceled_quoting_orders.result.TryFirst(); + Assert.IsTrue(canceled_quoting_order.HasValue, "order not found"); + Assert.IsTrue(canceled_quoting_order.Value.State.Status == OrderStatus.Canceled, "Quoting order may not have been canceled"); + + + // Spreader Order 2 + var spreader_order2_response = await OrderUtilities.send_order(_order_service, order1_def, -0.30, 1); + + var spreader_orders2 = await OrderUtilities.wait_for_filtered_orders(_order_service, + o => o.Where(ordi => ordi.Key == spreader_order2_response.value.OrderKey).Count() > 0, + ord => ord.Where(ordi => ordi.Key == spreader_order2_response.value.OrderKey)); + Assert.IsTrue(spreader_orders2.result.Count() > 0, "Can't find spreader order 2"); + + var working_spreader_order2 = spreader_orders2.result.TryFirst(); + Assert.IsTrue(working_spreader_order2.HasValue, "Spreader order not found"); + Assert.IsTrue(working_spreader_order2.Value.State.Status == OrderStatus.Working, "Spreader order 2 is not Working"); + + var spreader_order2_change_response = await OrderUtilities.modify_order(_order_service, working_spreader_order2.Value, -0.6); + var spreader_orders2_after_change = await OrderUtilities.wait_for_filtered_orders(_order_service, + o => o.Where(ordi => ordi.Key == spreader_order2_response.value.OrderKey && ordi.State.Price == -0.6).Count() > 0, + ord => ord.Where(ordi => ordi.Key == spreader_order2_response.value.OrderKey)); + Assert.IsTrue(spreader_orders2_after_change.result.Count() > 0, "Can't find spreader order 2 after first price change"); + + var spreader_order2_after_change = spreader_orders2_after_change.result.TryFirst(); + Assert.IsTrue(spreader_order2_after_change.HasValue, "order not found"); + Assert.IsTrue(spreader_order2_after_change.Value.State.Price == -0.6, "Failed to change price of spreader order"); + + var changed_quoting_order_price = 18.9; + + var changed_quoting_orders = await OrderUtilities.wait_for_filtered_orders(_order_service, + o => o.Where(ordi => ordi.ParentKey == spreader_order2_response.value.OrderKey && Math.Truncate(ordi.State.Price * 10.0) == 10.0 * changed_quoting_order_price).Count() > 0, + ord => ord.Where(ordi => ordi.ParentKey == spreader_order2_response.value.OrderKey), + TimeSpan.FromSeconds(30)); + + + + var changed_quoting_order = changed_quoting_orders.result.TryFirst(); + Assert.IsTrue(changed_quoting_order.HasValue, "order not found"); + Assert.IsTrue(Math.Truncate(changed_quoting_order.Value.State.Price * 10.0) == 10.0 * changed_quoting_order_price, "Quoting order has the wrong price"); + + + //fill spreader order + var spreader_order2_fill_change_response = await OrderUtilities.modify_order(_order_service, working_spreader_order2.Value, -0.2); + var spreader_orders2_fill_after_change = await OrderUtilities.wait_for_filtered_orders(_order_service, + o => o.Where(ordi => ordi.Key == spreader_order2_response.value.OrderKey && ordi.State.Price == -0.2).Count() > 0, + ord => ord.Where(ordi => ordi.Key == spreader_order2_response.value.OrderKey)); + Assert.IsTrue(spreader_orders2_fill_after_change.result.Count() > 0, "Can't find spreader order 2 after first price change"); + + var spreader_order2_fill_after_change = spreader_orders2_fill_after_change.result.TryFirst(); + Assert.IsTrue(spreader_order2_fill_after_change.HasValue, "order not found"); + Assert.IsTrue(spreader_order2_fill_after_change.Value.State.Price == -0.2, "Failed to change price of spreader order"); + + var filled_quoting_order_price = 18.5; + + var filled_quoting_orders = await OrderUtilities.wait_for_filtered_orders(_order_service, + o => o.Where(ordi => ordi.ParentKey == spreader_order2_response.value.OrderKey && Math.Truncate(ordi.State.Price * 10.0) == 10.0 * filled_quoting_order_price).Count() > 0, + ord => ord.Where(ordi => ordi.ParentKey == spreader_order2_response.value.OrderKey && Math.Truncate(ordi.State.Price * 10.0) == 10.0 * filled_quoting_order_price), + TimeSpan.FromSeconds(30)); + + var filled_quoting_order = filled_quoting_orders.result.TryFirst(); + Assert.IsTrue(filled_quoting_order.HasValue, "order not found"); + Assert.IsTrue(Math.Truncate(filled_quoting_order.Value.State.Price * 10.0) == 10.0 * filled_quoting_order_price, "Quoting order has the wrong price"); + + var hedging_order_price = 18.3; + + var hedging_orders = await OrderUtilities.wait_for_filtered_orders(_order_service, + o => o.Where(ordi => ordi.ParentKey == spreader_order2_response.value.OrderKey && Math.Truncate(ordi.State.Price * 10.0) == 10.0 * hedging_order_price).Count() > 0, + ord => ord.Where(ordi => ordi.ParentKey == spreader_order2_response.value.OrderKey && Math.Truncate(ordi.State.Price * 10.0) == 10.0 * hedging_order_price), + TimeSpan.FromSeconds(30)); + + var hedging_order = hedging_orders.result.TryFirst(); + Assert.IsTrue(hedging_order.HasValue, "order not found"); + Assert.IsTrue(Math.Truncate(hedging_order.Value.State.Price * 10.0) == 10.0 * hedging_order_price, "Hedging order has the wrong price"); + + } } diff --git a/testing/manual/spreader_misc/simple_sanity.txt b/testing/manual/spreader_misc/simple_sanity.txt index 470c110a4..c7164024b 100644 --- a/testing/manual/spreader_misc/simple_sanity.txt +++ b/testing/manual/spreader_misc/simple_sanity.txt @@ -3,11 +3,10 @@ This test makes sure the basic spreader functionality works for 2 legged synthet 0. Follow the instructions from docs/spreader_common_setup_1.txt 1. Enter a synthetic buy order of 1 lot at -0.30. Verify that a quoting order of 18.60 shows up in the second leg. + Cancel the synthetic order and the quoting order should be cancelled too. -2. Enter a synthetic buy order of 1 lot at -0.30, again. Now drag the the synthetic order down the ladder -(ie lower bid prices) to a few random prices of your choice. Verify that the quoter leg price is adjusted -accordingly and is correct. +2. Enter a synthetic buy order of 1 lot at -0.30, again. Now drag the the synthetic order down the ladder (ie lower bid prices) to a few random prices of your choice. Verify that the quoter leg price is adjusted accordingly and is correct. (If you have followed the instructions in common_setup_1.txt: Leg_1 will buy at 18.300 , and leg_2 will be your quoting leg) @@ -16,4 +15,5 @@ spread_price == price_mult_1 * leg_1 + price_mult_2 * leg_2 leg_2 == price_mult_1 * leg_1 - spread_price / -1.0 * price_mult_2 3. Now cross the synthetic order by dragging it all the way to -0.20 (there should be a 1 lot at that price). + Verify that the synthetic order gets filled at -0.20, the quoter order at 18.50, and the hedger order at 18.30. \ No newline at end of file From 7a4501378203c5df4f4546d59c5b4d9ae2945f0d Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Fri, 10 May 2019 13:49:37 -0400 Subject: [PATCH 60/85] Changed cover to stick around until all hedges are completed. --- .../orders/algo/algo_orders/CoverAlgo.cpp | 157 ++++++++++++------ .../orders/algo/algo_orders/CoverAlgo.h | 3 + 2 files changed, 111 insertions(+), 49 deletions(-) diff --git a/src/lib/orderman/orders/algo/algo_orders/CoverAlgo.cpp b/src/lib/orderman/orders/algo/algo_orders/CoverAlgo.cpp index 1ee5e14bc..01dff07fb 100644 --- a/src/lib/orderman/orders/algo/algo_orders/CoverAlgo.cpp +++ b/src/lib/orderman/orders/algo/algo_orders/CoverAlgo.cpp @@ -10,6 +10,18 @@ namespace orderman { using namespace api::orderman; +enum class CoverAlgo::Status { WORKING, WAITING_FOR_HEDGES, DONE }; +const char *CoverAlgo::status_name(Status st) { + switch (st) { + case Status::WORKING: + return "WORKING"; + case Status::WAITING_FOR_HEDGES: + return "WAITING_FOR_HEDGES"; + case Status::DONE: + return "Done"; + } + return nullptr; +} struct CoverAlgo::PImpl { IAlgoServices *algo; OrderKey child_key = null_order_key(); @@ -17,6 +29,8 @@ struct CoverAlgo::PImpl { ChildTemplate *child_template = nullptr; ChildTemplate *hedge_template = nullptr; Quantity hedged_fills = Quantity(0); + Status status = Status::WORKING; + std::vector hedges; }; CoverAlgo::CoverAlgo() : me(new PImpl()) {} @@ -77,64 +91,96 @@ void CoverAlgo::do_launch_attached(IAlgoServices &algo, OrderKey k, } void CoverAlgo::on_child_change(OrderKey k, ChildState const &st) { - bts_assert(k == me->child_key); // We are not subscribed to hedge changes + if (k == me->child_key) { + if (st.total_fills.value() > me->hedged_fills.value()) { + auto hedge_qty = + Quantity(st.total_fills.value() - me->hedged_fills.value()); + auto hedge_price = securities::improve_price( + algo().definition().side(), st.last_price, + me->price_improvement_delta); - if (st.total_fills.value() > me->hedged_fills.value()) { - auto hedge_qty = - Quantity(st.total_fills.value() - me->hedged_fills.value()); - auto hedge_price = - securities::improve_price(algo().definition().side(), st.last_price, - me->price_improvement_delta); + auto hedge = + algo().launch_child(*me->hedge_template, hedge_price, hedge_qty); + me->hedges.push_back(hedge); - algo().launch_child(*me->hedge_template, hedge_price, hedge_qty, - false); // do not subscribe to hedge changes - - me->hedged_fills = Quantity(hedge_qty.value() + me->hedged_fills.value()); - LOG(OrderMan, Diag, - "CoverAlgo {}, launched new hedge in response to child fill. Hedge: " - "{} {}@{}", - algo().order_key().value(), - securities::side_name(algo().definition().side()), hedge_qty.value(), - hedge_price.value()); - } - algo().update_fill_state(st.total_fills, st.last_price); - switch (st.status) { - case ChildStatus::WORKING: - return; - case ChildStatus::COMPLETED: - algo().detach_all_children(); - algo().complete(); - return; - case ChildStatus::CANCELED: - algo().terminate_error("Unexpected child cancel"); - return; + me->hedged_fills = + Quantity(hedge_qty.value() + me->hedged_fills.value()); + LOG(OrderMan, Diag, + "CoverAlgo {}, launched new hedge in response to child fill. " + "Hedge: " + "{} {}@{}", + algo().order_key().value(), + securities::side_name(algo().definition().side()), + hedge_qty.value(), hedge_price.value()); + } + algo().update_fill_state(st.total_fills, st.last_price); + switch (st.status) { + case ChildStatus::WORKING: + break; + case ChildStatus::COMPLETED: + LOG(OrderMan, Info, + "CoverAlgo {}. Child is completed, waiting for hedges ", + algo().order_key().value()); + me->status = Status::WAITING_FOR_HEDGES; + break; + case ChildStatus::CANCELED: + LOG(OrderMan, Info, "CoverAlgo {}. Child was canceled, terminating", + algo().order_key().value()); + me->status = Status::DONE; + algo().terminate_error("Unexpected child cancel"); + break; + } + } else { + switch (st.status) { + case ChildStatus::WORKING: + break; + case ChildStatus::COMPLETED: + if (auto it = std::find(me->hedges.begin(), me->hedges.end(), k); + it != me->hedges.end()) + me->hedges.erase(it); + break; + case ChildStatus::CANCELED: + LOG(OrderMan, Info, "CoverAlgo {}. Hedge was canceled, terminating", + algo().order_key().value()); + me->status = Status::DONE; + algo().terminate_error("Unexpected child cancel"); + break; + } } + check_terminate(); } void CoverAlgo::do_update(api::orderman::Update const &update) { - LOG(OrderMan, Diag, "CoverAlgo order {} receiving update: {}", - algo().order_key().value(), utils::pb::to_string(update)); + if (me->status == Status::WORKING) { + LOG(OrderMan, Diag, "CoverAlgo order {} receiving update: {}", + algo().order_key().value(), utils::pb::to_string(update)); - if (update.has_custom() && !update.custom().has_cover()) { - algo().terminate_error("Invalid update"); - return; - } - - Update child_update; - child_update.set_price(update.price()); - child_update.set_quantity(update.quantity()); - - // process custom part of the udpate if any - if (update.has_custom()) { - auto &cover_update = update.custom().cover(); - set_price_improvement_delta(algo(), cover_update); - if (cover_update.has_child()) { - *child_update.mutable_custom() = cover_update.child(); + if (update.has_custom() && !update.custom().has_cover()) { + algo().terminate_error("Invalid update"); + return; } - if (cover_update.has_hedge()) - init_hedge_template(algo(), cover_update.hedge()); + + Update child_update; + child_update.set_price(update.price()); + child_update.set_quantity(update.quantity()); + + // process custom part of the udpate if any + if (update.has_custom()) { + auto &cover_update = update.custom().cover(); + set_price_improvement_delta(algo(), cover_update); + if (cover_update.has_child()) { + *child_update.mutable_custom() = cover_update.child(); + } + if (cover_update.has_hedge()) + init_hedge_template(algo(), cover_update.hedge()); + } + algo().update_child(me->child_key, child_update); + } else { + LOG(OrderMan, Info, + "CoverAlgo {}. Received update after child has completed. Ignoring " + "it.", + algo().order_key().value()); } - algo().update_child(me->child_key, child_update); } void orderman::CoverAlgo::init_hedge_template( @@ -155,5 +201,18 @@ void CoverAlgo::set_price_improvement_delta( securities::tick_size(serv.security()).value() * cover.hedge_ticks()); } +void CoverAlgo::check_terminate() { + LOG(OrderMan, Diag, + "CoverAlgo {} check_terminate. State: {} Hedge count: {}", + algo().order_key(), log::CharPtrWrapper{status_name(me->status)}, + me->hedges.size()); + + if (me->status == Status::WAITING_FOR_HEDGES && me->hedges.empty()) { + LOG(OrderMan, Info, "CoverAlgo {}. Last hedge completed, terminating", + algo().order_key().value()); + algo().complete(); + } +} + } // namespace orderman } // namespace bts diff --git a/src/lib/orderman/orders/algo/algo_orders/CoverAlgo.h b/src/lib/orderman/orders/algo/algo_orders/CoverAlgo.h index 3e8d9adbf..3bedb00e7 100644 --- a/src/lib/orderman/orders/algo/algo_orders/CoverAlgo.h +++ b/src/lib/orderman/orders/algo/algo_orders/CoverAlgo.h @@ -21,6 +21,7 @@ public: private: struct PImpl; + enum class Status; std::unique_ptr me; IAlgoServices &algo() const; void @@ -28,6 +29,8 @@ private: api::orderman::CoverOrderDefinition const &); void init_hedge_template(IBeforeLaunchAlgoServices &, api::orderman::CustomOrderDefinition const &); + void check_terminate(); + const char *status_name(Status st); }; } // namespace orderman From 70af046131a53021c55e3e46e54960ffbcb6f233 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Fri, 10 May 2019 13:51:04 -0400 Subject: [PATCH 61/85] Updated docker subrepo ptr --- docker/linux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/linux b/docker/linux index ce08a5129..e13bb5d8d 160000 --- a/docker/linux +++ b/docker/linux @@ -1 +1 @@ -Subproject commit ce08a5129f14af0d1bb9fa569d1e721b694964fe +Subproject commit e13bb5d8df14171a9a5abfdb8b6b4ecbc519032a From 689bfa1c7667d84941c2975c4da6a25957eb7aa5 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Fri, 10 May 2019 13:51:25 -0500 Subject: [PATCH 62/85] OrderBook 'attach popup' price/qty vis depends on the selected attach-algo (not the selected orders) --- .../Modules/Trading/OrderBook/OrderBookVM.cs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs index 4a0fcb5e1..55c1f0f53 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs @@ -275,15 +275,14 @@ namespace mbts.spark { Disposable.replace(ref attach_vm_hooks, Disposable.Create( connect_common_selected_params(attach_vm), - _selected_params_mon.subscribe_algos().Subscribe(algos => { + attach_vm.algo_vm.observe_algo_root().Subscribe(algo_root => { bool show_price, show_size; - using (var en = algos.Select(a => a.actual().CustomCase).Distinct().GetEnumerator()) { - if (show_price = show_size = en.MoveNext()) { - do { - if (show_price) show_price = AttachConf.show_price(en.Current); - if (show_size) show_size = AttachConf.show_size(en.Current); - } while ((show_price || show_size) && en.MoveNext()); - } + if (algo_root == null) { + show_price = false; + show_size = false; + } else { + show_price = AttachConf.show_price(algo_root.order_type); + show_size = AttachConf.show_size(algo_root.order_type); } attach_vm.price_permissions = bool_to_perm(show_price); attach_vm.size_permissions = bool_to_perm(show_size); From 02752d9151fffd47178da60e96fffc3ad5805e4d Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Fri, 10 May 2019 16:14:06 -0400 Subject: [PATCH 63/85] SPARK-2064: simple sanity simplified --- src/client/spark_test/System/SpreaderTests.cs | 134 +++++++----------- 1 file changed, 53 insertions(+), 81 deletions(-) diff --git a/src/client/spark_test/System/SpreaderTests.cs b/src/client/spark_test/System/SpreaderTests.cs index a752f6cae..10a8a42d5 100644 --- a/src/client/spark_test/System/SpreaderTests.cs +++ b/src/client/spark_test/System/SpreaderTests.cs @@ -72,109 +72,81 @@ namespace bts.tests test_account, test_user ); + var spreader_order1_response = await OrderUtilities.send_order(_order_service, order1_def, -0.30, 1); + Assert.IsTrue(spreader_order1_response.success, "Failed to send first spreader order"); - var spreader_orders1 = await OrderUtilities.wait_for_filtered_orders(_order_service, - o => o.Where(ordi => ordi.Key == spreader_order1_response.value.OrderKey).Count() > 0, - ord => ord.Where(ordi => ordi.Key == spreader_order1_response.value.OrderKey)); - Assert.IsTrue(spreader_orders1.result.Count() > 0, "Can't find spreader order"); + var (spreader_order_is_working, working_spreader_order1) = await verify_order_status(spreader_order1_response.value.OrderKey, OrderStatus.Working); + Assert.IsTrue(spreader_order_is_working, "Spreader order is not Working"); - var working_spreader_order1 = spreader_orders1.result.Where(ordi => ordi.Key == spreader_order1_response.value.OrderKey).TryFirst(); - Assert.IsTrue(working_spreader_order1.HasValue, "Spreader order not found"); - Assert.IsTrue(working_spreader_order1.Value.State.Status == OrderStatus.Working, "Spreader order is not Working"); + var (quoting_order_is_working, working_quoting_order) = await verify_child_order_status(spreader_order1_response.value.OrderKey, OrderStatus.Working); + Assert.IsTrue(quoting_order_is_working, "Order is not Working"); + Assert.IsTrue(working_quoting_order.Value.State.Price == 18.6, "Quoting order has the wrong price"); - var quoting_orders = await OrderUtilities.wait_for_filtered_orders(_order_service, - o => o.Where(ordi => ordi.ParentKey == spreader_order1_response.value.OrderKey).Count() > 0, - ord => ord.Where(ordi => ordi.ParentKey == spreader_order1_response.value.OrderKey)); - - var quoting_order = quoting_orders.result.TryFirst(); - Assert.IsTrue(quoting_order.HasValue, "order not found"); - Assert.IsTrue(quoting_order.Value.State.Status == OrderStatus.Working, "Order is not Working"); - Assert.IsTrue(quoting_order.Value.State.Price == 18.6, "Quoting order has the wrong price"); - - var spreader_order1_cancel_ret = await OrderUtilities.cancel_orders(_order_service, new List {working_spreader_order1.Value}); - - var canceled_quoting_orders = await OrderUtilities.wait_for_filtered_orders(_order_service, - o => o.Where(ordi => ordi.ParentKey == spreader_order1_response.value.OrderKey && ordi.State.Status == OrderStatus.Canceled).Count() > 0, - ord => ord.Where(ordi => ordi.ParentKey == spreader_order1_response.value.OrderKey), - TimeSpan.FromSeconds(30)); - - var canceled_quoting_order = canceled_quoting_orders.result.TryFirst(); - Assert.IsTrue(canceled_quoting_order.HasValue, "order not found"); - Assert.IsTrue(canceled_quoting_order.Value.State.Status == OrderStatus.Canceled, "Quoting order may not have been canceled"); + Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, new List {working_spreader_order1.Value}), "Failed to cancel order"); + var (quoting_order_is_canceled, canceled_quoting_order) = await verify_child_order_status(spreader_order1_response.value.OrderKey, OrderStatus.Canceled); + Assert.IsTrue(quoting_order_is_canceled, "Quoting order may not have been canceled"); // Spreader Order 2 var spreader_order2_response = await OrderUtilities.send_order(_order_service, order1_def, -0.30, 1); + Assert.IsTrue(spreader_order2_response.success, "Failed to send spreader order 2"); - var spreader_orders2 = await OrderUtilities.wait_for_filtered_orders(_order_service, - o => o.Where(ordi => ordi.Key == spreader_order2_response.value.OrderKey).Count() > 0, - ord => ord.Where(ordi => ordi.Key == spreader_order2_response.value.OrderKey)); - Assert.IsTrue(spreader_orders2.result.Count() > 0, "Can't find spreader order 2"); + var (spreader_order2_is_working, working_spreader_order2) = await verify_order_status(spreader_order2_response.value.OrderKey, OrderStatus.Working); - var working_spreader_order2 = spreader_orders2.result.TryFirst(); - Assert.IsTrue(working_spreader_order2.HasValue, "Spreader order not found"); - Assert.IsTrue(working_spreader_order2.Value.State.Status == OrderStatus.Working, "Spreader order 2 is not Working"); + Assert.IsTrue(spreader_order2_is_working, "Spreader order 2 is not 'Working'"); var spreader_order2_change_response = await OrderUtilities.modify_order(_order_service, working_spreader_order2.Value, -0.6); - var spreader_orders2_after_change = await OrderUtilities.wait_for_filtered_orders(_order_service, - o => o.Where(ordi => ordi.Key == spreader_order2_response.value.OrderKey && ordi.State.Price == -0.6).Count() > 0, - ord => ord.Where(ordi => ordi.Key == spreader_order2_response.value.OrderKey)); - Assert.IsTrue(spreader_orders2_after_change.result.Count() > 0, "Can't find spreader order 2 after first price change"); + Assert.IsTrue(spreader_order2_change_response.success, "failed to modify spreader order"); - var spreader_order2_after_change = spreader_orders2_after_change.result.TryFirst(); - Assert.IsTrue(spreader_order2_after_change.HasValue, "order not found"); - Assert.IsTrue(spreader_order2_after_change.Value.State.Price == -0.6, "Failed to change price of spreader order"); - - var changed_quoting_order_price = 18.9; - - var changed_quoting_orders = await OrderUtilities.wait_for_filtered_orders(_order_service, - o => o.Where(ordi => ordi.ParentKey == spreader_order2_response.value.OrderKey && Math.Truncate(ordi.State.Price * 10.0) == 10.0 * changed_quoting_order_price).Count() > 0, - ord => ord.Where(ordi => ordi.ParentKey == spreader_order2_response.value.OrderKey), - TimeSpan.FromSeconds(30)); - - - - var changed_quoting_order = changed_quoting_orders.result.TryFirst(); - Assert.IsTrue(changed_quoting_order.HasValue, "order not found"); - Assert.IsTrue(Math.Truncate(changed_quoting_order.Value.State.Price * 10.0) == 10.0 * changed_quoting_order_price, "Quoting order has the wrong price"); + Assert.IsTrue(await verify_order_price(spreader_order2_response.value.OrderKey, -0.6), "Failed to change price of spreader order"); + // verify quoting order price + Assert.IsTrue(await verify_child_order_price(spreader_order2_response.value.OrderKey, 18.9), "Quoting order incorrect or not found"); //fill spreader order var spreader_order2_fill_change_response = await OrderUtilities.modify_order(_order_service, working_spreader_order2.Value, -0.2); - var spreader_orders2_fill_after_change = await OrderUtilities.wait_for_filtered_orders(_order_service, - o => o.Where(ordi => ordi.Key == spreader_order2_response.value.OrderKey && ordi.State.Price == -0.2).Count() > 0, - ord => ord.Where(ordi => ordi.Key == spreader_order2_response.value.OrderKey)); - Assert.IsTrue(spreader_orders2_fill_after_change.result.Count() > 0, "Can't find spreader order 2 after first price change"); + Assert.IsTrue(spreader_order2_fill_change_response.success, "failed to modify spreader order"); - var spreader_order2_fill_after_change = spreader_orders2_fill_after_change.result.TryFirst(); - Assert.IsTrue(spreader_order2_fill_after_change.HasValue, "order not found"); - Assert.IsTrue(spreader_order2_fill_after_change.Value.State.Price == -0.2, "Failed to change price of spreader order"); + Assert.IsTrue(await verify_order_price(spreader_order2_response.value.OrderKey, -0.2), "Failed to change price of spreader order"); - var filled_quoting_order_price = 18.5; - - var filled_quoting_orders = await OrderUtilities.wait_for_filtered_orders(_order_service, - o => o.Where(ordi => ordi.ParentKey == spreader_order2_response.value.OrderKey && Math.Truncate(ordi.State.Price * 10.0) == 10.0 * filled_quoting_order_price).Count() > 0, - ord => ord.Where(ordi => ordi.ParentKey == spreader_order2_response.value.OrderKey && Math.Truncate(ordi.State.Price * 10.0) == 10.0 * filled_quoting_order_price), - TimeSpan.FromSeconds(30)); - - var filled_quoting_order = filled_quoting_orders.result.TryFirst(); - Assert.IsTrue(filled_quoting_order.HasValue, "order not found"); - Assert.IsTrue(Math.Truncate(filled_quoting_order.Value.State.Price * 10.0) == 10.0 * filled_quoting_order_price, "Quoting order has the wrong price"); - - var hedging_order_price = 18.3; - - var hedging_orders = await OrderUtilities.wait_for_filtered_orders(_order_service, - o => o.Where(ordi => ordi.ParentKey == spreader_order2_response.value.OrderKey && Math.Truncate(ordi.State.Price * 10.0) == 10.0 * hedging_order_price).Count() > 0, - ord => ord.Where(ordi => ordi.ParentKey == spreader_order2_response.value.OrderKey && Math.Truncate(ordi.State.Price * 10.0) == 10.0 * hedging_order_price), - TimeSpan.FromSeconds(30)); - - var hedging_order = hedging_orders.result.TryFirst(); - Assert.IsTrue(hedging_order.HasValue, "order not found"); - Assert.IsTrue(Math.Truncate(hedging_order.Value.State.Price * 10.0) == 10.0 * hedging_order_price, "Hedging order has the wrong price"); + // verify quoting order price + Assert.IsTrue(await verify_child_order_price(spreader_order2_response.value.OrderKey, 18.5), "Quoting order incorrect or not found"); + // verify Hedging order price + Assert.IsTrue(await verify_child_order_price(spreader_order2_response.value.OrderKey, 18.3), "Hedging order incorrect or not found"); } + private async Task<(Boolean, OrderRecordImmutable?)> verify_child_order_status(uint parent_order_key, OrderStatus expected_status) { + var order = await get_order(ordi => ordi.ParentKey == parent_order_key && ordi.State.Status == expected_status); + return (order.HasValue && order.Value.State.Status == expected_status, order); + } + private async Task<(Boolean, OrderRecordImmutable?)> verify_order_status(uint order_key, OrderStatus expected_status) { + var order = await get_order(ordi => ordi.Key == order_key); + return (order.HasValue && order.Value.State.Status == expected_status, order); + } + + private async Task verify_order_price(uint order_key, double expected_price, double precision = 10) { + var order = await get_order(ordi => ordi.Key == order_key && Math.Truncate(ordi.State.Price * precision) == precision * expected_price); + return order.HasValue && (Math.Truncate(order.Value.State.Price * precision) == precision * expected_price); + } + + private async Task verify_child_order_price(uint parent_order_key, double expected_price, double precision = 10.0) + { + var order = await get_order(ordi => ordi.ParentKey == parent_order_key && Math.Truncate(ordi.State.Price * precision) == precision * expected_price); + return order.HasValue && (Math.Truncate(order.Value.State.Price * precision) == precision * expected_price); + } + + private async Task get_order(Func predicate) + { + var orders = await OrderUtilities.wait_for_filtered_orders(_order_service, + o => o.Where(predicate).Count() > 0, + ord => ord.Where(predicate), + TimeSpan.FromSeconds(30)); + return orders.result.TryFirst(); + } + + } } \ No newline at end of file From 6c5429b3538132882d9bcb7d4b47328ce8d8b687 Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Fri, 10 May 2019 16:36:13 -0400 Subject: [PATCH 64/85] Minor updates to position limits test --- src/client/spark_test/System/LimitsTests.cs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/client/spark_test/System/LimitsTests.cs b/src/client/spark_test/System/LimitsTests.cs index 9ef678e81..0eb96bf24 100644 --- a/src/client/spark_test/System/LimitsTests.cs +++ b/src/client/spark_test/System/LimitsTests.cs @@ -13,6 +13,7 @@ using bts.api.orderman; using mbts; using mbts.tests; using bts.co2; +using bts.tests.utils; namespace bts.tests { @@ -79,21 +80,24 @@ namespace bts.tests return securities; } - [Test(Timeout = 30)] + [Test(Timeout = 300)] public async Task test_position_limits() { + bool system_ready = await TestUtils.test_for_system_ready(_context, TimeSpan.FromMinutes(5)); + Assert.IsTrue(system_ready, "System is not ready"); + string test_symbol = "CL"; var test_user = ( name: "bts", id: 0u ); var test_account = ( name: "ACC A", id: 0u ); - BTSAccountImmutable? account_data = await AdminUtilities.get_account_data(_admin_service, test_account.name); + var account_task = AdminUtilities.get_account_data(_admin_service, test_account.name); + BTSAccountImmutable? account_data = await TestUtils.await_with_timeout(account_task, TimeSpan.FromMinutes(0.5)); + Assert.IsTrue(account_data.HasValue); test_account.id = account_data.Value.Id; - var user_data = await AdminUtilities.get_user_data(_admin_service, test_user.name); Assert.IsTrue(user_data.HasValue); test_user.id = user_data.Value.Id; - var clip_set_success = await AdminUtilities.set_clip_limit(_admin_service, 10000, account_data, test_symbol); Assert.IsTrue(clip_set_success, "Failed to set clip limit "); @@ -114,6 +118,7 @@ namespace bts.tests IEnumerable no_limit_filtered_orders = await get_filtered_orders(x => x.Key == no_limit_order_result.value.OrderKey); + Assert.IsNotNull(no_limit_filtered_orders, "No orders retrieved"); Assert.IsTrue(no_limit_filtered_orders.First().State.Status == OrderStatus.RejectedBySpark, "'Absent Limit' Order is not 'Rejected by Spark'"); bool position_set_success = await AdminUtilities.set_position_limit(_admin_service, test_account.name, test_symbol, 60, 120); @@ -307,7 +312,7 @@ namespace bts.tests } await Task.Delay(100); } - while(it < 100 &&(_orders == null || current_count < count)); + while(it < 100 && (_orders == null || current_count < count)); return null; From 04fdb9626daeac5234732f0f80ed757529379d7c Mon Sep 17 00:00:00 2001 From: John Colvin Date: Fri, 10 May 2019 15:57:31 -0500 Subject: [PATCH 65/85] SPARK-2409 Changing a composite algo name does not reflect in the attach window dropdown --- .../AlgoStudio/Utils/AlgoCollectionHelper.cs | 14 +++++++++----- .../Trading/AlgoStudio/VM/AlgoDefinitionVM.cs | 1 + .../Trading/AlgoStudio/VM/AlgoVMWrapper.cs | 17 ++++++++--------- .../Modules/Trading/OrderBook/OrderBookVM.cs | 2 +- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/client/spark/Modules/Trading/AlgoStudio/Utils/AlgoCollectionHelper.cs b/src/client/spark/Modules/Trading/AlgoStudio/Utils/AlgoCollectionHelper.cs index 66219f985..b66b82839 100644 --- a/src/client/spark/Modules/Trading/AlgoStudio/Utils/AlgoCollectionHelper.cs +++ b/src/client/spark/Modules/Trading/AlgoStudio/Utils/AlgoCollectionHelper.cs @@ -47,15 +47,19 @@ namespace mbts.spark { } } - public static void update_persisted_definition_wrappers(ObservableKeyedCollection target, TableUpdate update, Func factory) { + public static void update_persisted_definition_wrappers(IDictionary target, TableUpdate update, Func factory) { foreach (var d in update.removed) target.Remove(d.key); - foreach (var d in update.changed) - target[d.key].load_from(d.value); + foreach (var d in update.changed) { + var vm = target[d.key]; + if (vm is AlgoVMWrapper wrapper) { + wrapper.value.load(d); + } else vm.load_from(d.value); + } foreach (var d in update.added) { if (target.TryGetValue(d.key, out IAlgoVM def)) { def.load_from(d.value); - } else target.Add(new AlgoVMWrapper(factory(d))); + } else target.Add(d.key, new AlgoVMWrapper(factory(d))); } } @@ -114,4 +118,4 @@ namespace mbts.spark { DisplayNameHelper.compare_alpha_numeric(a.display_name, b.display_name); } } -} \ No newline at end of file +} diff --git a/src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoDefinitionVM.cs b/src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoDefinitionVM.cs index 008e6cbd3..7ec794043 100644 --- a/src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoDefinitionVM.cs +++ b/src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoDefinitionVM.cs @@ -202,6 +202,7 @@ namespace mbts.spark { public string name { get => _name.Value; set => _name.set(value); } public string abbreviation { get => _abbreviation.Value; set => _abbreviation.set(value); } public ObservableCollection algo_roots => _algo_roots; + public IDictionary algo_roots_index => _algo_roots; public IReadOnlyObservableCollection algos_view { get; } public IAlgoVM algo_root { get => _algo_root.Value; set => _algo_root.set(value); } public IAlgoVM visible_algo_root => _visible_algo_root.Value; diff --git a/src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoVMWrapper.cs b/src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoVMWrapper.cs index 88cac97ec..aa97940d2 100644 --- a/src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoVMWrapper.cs +++ b/src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoVMWrapper.cs @@ -29,13 +29,12 @@ namespace mbts.spark { nameof(editing_permissions), nameof(definition), nameof(display_name) - }, - _value_echo_props = new HashSet { - nameof(AlgoDefinitionVM.name), - nameof(AlgoDefinitionVM.abbreviation), - nameof(AlgoDefinitionVM.badge_fill), - nameof(AlgoDefinitionVM.can_undo), - nameof(AlgoDefinitionVM.can_redo) + }; + static readonly IReadOnlyDictionary + _value_echo_props = new Dictionary { + { nameof(AlgoDefinitionVM.name), nameof(display_name) }, + { nameof(AlgoDefinitionVM.abbreviation), nameof(abbreviation) }, + { nameof(AlgoDefinitionVM.badge_fill), nameof(meta) } }; readonly ObservableAggregateCollection _child_algo_selectors = new ObservableAggregateCollection(); readonly ObservableAggregateCollection _algo_params = new ObservableAggregateCollection(); @@ -143,8 +142,8 @@ namespace mbts.spark { RaisePropertyChanged(e.PropertyName); } void on_value_prop_changed(object sender, PropertyChangedEventArgs e) { - if (_value_echo_props.Contains(e.PropertyName)) - RaisePropertyChanged(e.PropertyName); + if (_value_echo_props.TryGetValue(e.PropertyName, out string wrapper_prop)) + RaisePropertyChanged(wrapper_prop); } } diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs index 55c1f0f53..2364208b9 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs @@ -291,7 +291,7 @@ namespace mbts.spark { }), // custom templates (created by user) _model.algo_model.definitions.OnDispatcher().Subscribe(u => - AlgoCollectionHelper.update_persisted_definition_wrappers(attach_vm.algo_vm.algo_roots as ObservableKeyedCollection, u, mk_factory(true))))); + AlgoCollectionHelper.update_persisted_definition_wrappers(attach_vm.algo_vm.algo_roots_index, u, mk_factory(true))))); // vm factory Func mk_factory(bool connect) => fd => { From 5e1a91f484094789f2775020217bd78d136727f1 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Fri, 10 May 2019 16:01:07 -0500 Subject: [PATCH 66/85] add period to the end of 'Make current parameters the local defaults' tooltip --- src/client/spark/view/Controls/Order/AlgoTemplateToolBar.xaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/view/Controls/Order/AlgoTemplateToolBar.xaml b/src/client/spark/view/Controls/Order/AlgoTemplateToolBar.xaml index c07dd5593..f6cf894af 100644 --- a/src/client/spark/view/Controls/Order/AlgoTemplateToolBar.xaml +++ b/src/client/spark/view/Controls/Order/AlgoTemplateToolBar.xaml @@ -243,7 +243,7 @@ - Make current paramaters the local defaults + Make current paramaters the local defaults. These values will be lost when the ladder closes. From a8610beafd5cf45b60e8898383267ad0f0602f0e Mon Sep 17 00:00:00 2001 From: John Colvin Date: Fri, 10 May 2019 20:38:33 -0500 Subject: [PATCH 67/85] SPARK-2432 Crash when detaching order window --- src/client/spark/view/DragDrop/DragDropHelper.cs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/client/spark/view/DragDrop/DragDropHelper.cs b/src/client/spark/view/DragDrop/DragDropHelper.cs index efd4ce573..3fdc56532 100644 --- a/src/client/spark/view/DragDrop/DragDropHelper.cs +++ b/src/client/spark/view/DragDrop/DragDropHelper.cs @@ -106,12 +106,20 @@ namespace mbts.ui var effect = DragDrop.DoDragDrop(_source, data, DragDropEffects.Move); } catch (Exception error) { Log.Ui.warn("Dragdrop failed: " + error.ToString()); } - await show_document(_source, _source_window, true); - _source = null; - _source_window = null; + /// Reentrancy Risk: + /// During the await period for , + /// the new window hooks up to the dragdrop events via . + /// It must be assured that the new window does not get a callback into + /// and attempt to create a second for the same instance . + /// For this reason, is set to null before is called. _source_image = null; _target = null; _dddp = null; + var src = _source; + var src_win = _source_window; + _source = null; + _source_window = null; + await show_document(src, src_win, true); } } } From 762369ad2b64c350e06d217acd44144c5e78eee5 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Fri, 10 May 2019 21:22:42 -0500 Subject: [PATCH 68/85] SPARK-2433 child order font weight in OB should be dropped or adjusted --- .../Trading/OrderBook/OrderBookColumns.cs | 51 ++++++++++++++++--- src/lib/config/protos/UI.proto | 4 ++ 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs index ab74a045d..85f7222d4 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs @@ -566,7 +566,7 @@ namespace mbts.spark { public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) => CellRender.draw_text(dc, _get_glyph_context(row), font_size, _get_text(row), position, row_height, - font_brush, null, horizontal_alignment, VerticalAlignment.Center, DefaultCellPadding); + RenderUtils.get_foreground(row, font_brush), null, horizontal_alignment, VerticalAlignment.Center, DefaultCellPadding); } #region Explicit Column Defs @@ -598,12 +598,15 @@ namespace mbts.spark { } public class OrderBookOrderTypeColumn : OrderTypeColumn, ICVOrderBookColumn { + readonly BindingBase _foreground_binding; ListSortDirection? _cached_sort_direction; + public OrderBookOrderTypeColumn(Columns key, string badge_fill_binding_path, string display_name_binding_path, IComparer row_comparer, IDataGridColumnPersistentFilter filter, string header = DefaultColumnHeaders.OrderType) : base(key, new Binding(badge_fill_binding_path), new Binding(display_name_binding_path), filter, header) { id = key.ToString(); this.filter = filter; font_weight_binding = RenderUtils.get_font_weight_binding(); + _foreground_binding = RenderUtils.get_foreground_binding(this); _cached_sort_direction = SortDirection; this.comparer = row_comparer; } @@ -615,6 +618,12 @@ namespace mbts.spark { IDataGridColumnFilter ICVOrderBookColumn.filter => filter; ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; } + public override FrameworkElement view_factory() { + var view = base.view_factory(); + view.SetBinding(OrderTypeItem.ForegroundProperty, _foreground_binding); + return view; + } + protected override void OnSortDirectionChanged(ListSortDirection? new_value) { _cached_sort_direction = new_value; base.OnSortDirectionChanged(new_value); @@ -646,7 +655,7 @@ namespace mbts.spark { } public class SecurityColumn : FrameworkElementColumnBase, ICVOrderBookColumn, IFilterableColumn { - readonly BindingBase _font_weight_binding = RenderUtils.get_font_weight_binding(); + readonly BindingBase _font_weight_binding = RenderUtils.get_font_weight_binding(), _foreground_binding; readonly IValueConverter _expander_visiblilty_converter = new ui.converters.BooleanToVisibilityConverter { VisibilityForFalse = Visibility.Hidden }; readonly IValueConverter _margin_converter = new AnonymousValueConverter(level => new Thickness(10 * level, 0, 0, 0)); @@ -655,6 +664,7 @@ namespace mbts.spark { public SecurityColumn(IDataGridColumnPersistentFilter filter) : base(Columns.Security) { id = key.ToString(); this.filter = filter; + _foreground_binding = RenderUtils.get_foreground_binding(this); Header = DefaultColumnHeaders.Security; Width = DefaultColumnWidths.Security; _cached_sort_direction = SortDirection; @@ -681,7 +691,7 @@ namespace mbts.spark { security_view.SetBinding(TextBlock.FontSizeProperty, new Binding(nameof(font_size)) { Source = this }); security_view.SetBinding(TextBlock.TextProperty, nameof(OrderBookRowVM.security_text)); security_view.SetBinding(TextBlock.HorizontalAlignmentProperty, new Binding(nameof(horizontal_alignment)) { Source = this }); - security_view.SetBinding(TextBlock.ForegroundProperty, new Binding(nameof(font_brush)) { Source = this }); + security_view.SetBinding(TextBlock.ForegroundProperty, _foreground_binding); security_view.SetBinding(TextBlock.FontFamilyProperty, new Binding(nameof(font_family)) { Source = this }); security_view.SetBinding(TextBlock.FontWeightProperty, _font_weight_binding); // @@ -1153,19 +1163,24 @@ namespace mbts.spark { static class RenderUtils { public static FontWeight root_font_weight = FontWeights.Normal, - child_font_weight = FontWeights.Light; + child_font_weight = FontWeights.Normal; + static readonly IMultiValueConverter _foreground_converter = + ValueConverter.create(_get_foreground); public static double font_size = FontSizes.Small; public static readonly IValueConverter _sub_type_text_converter = new ui.spark.converters.OrderSubTypeDisplayTextConverter(), _side_brush_converter = new AnonymousValueConverter(s => s == Side.Buy ? BTSColors.MarketData.bid : BTSColors.MarketData.ask), - _font_weight_converter = new AnonymousValueConverter(is_child => is_child ? FontWeights.Light : FontWeights.Normal); + _font_weight_converter = new AnonymousValueConverter(is_child => is_child ? child_font_weight : root_font_weight); + public static readonly IMultiValueConverter _price_text_converter = new ui.spark.converters.TickDisplayConverter(), _background_converter = new AnonymousMultiValueConverter( (selected, status) => selected || status != OrderStatus.Working ? Brushes.Transparent : BTSColors.Orders.working_order_background_brush); + static readonly bool _animate_child_order_rows = mbts.config.Configuration.get().animate_child_order_rows; + public static BindingBase _get_background_binding() { var mb = new MultiBinding { Converter = _background_converter }; mb.Bindings.Add(new Binding(nameof(ListBoxItem.IsSelected)) { RelativeSource = new RelativeSource(RelativeSourceMode.FindAncestor, typeof(ListBoxItem), 1) }); @@ -1173,9 +1188,29 @@ namespace mbts.spark { return mb; } - public static Brush get_side_brush(OrderBookRowVM r) => r.side == Side.Buy ? BTSColors.MarketData.bid : BTSColors.MarketData.ask; - public static BindingBase get_font_weight_binding() => new Binding(nameof(OrderBookRowVM.is_child)) { Converter = _font_weight_converter }; - } + public static BindingBase get_foreground_binding(ColumnBase col) { + var col_font_brush_binding = new Binding(nameof(ColumnBase.font_brush)) { Source = col }; + if (!_animate_child_order_rows) + return col_font_brush_binding; + var mb = new MultiBinding { Converter = _foreground_converter }; + mb.Bindings.Add(new Binding(nameof(OrderBookRowVM.is_child))); + mb.Bindings.Add(col_font_brush_binding); + return mb; + } + public static Brush get_side_brush(OrderBookRowVM r) => get_foreground(r, r.side == Side.Buy ? BTSColors.MarketData.bid : BTSColors.MarketData.ask); + public static BindingBase get_font_weight_binding() => new Binding(nameof(OrderBookRowVM.is_child)) { Converter = _font_weight_converter }; + + public static Brush get_foreground(OrderBookRowVM row, Brush brush) => + _animate_child_order_rows ? _get_foreground(row.is_child, brush) : brush; + + static Brush _get_foreground(bool is_child, Brush brush) { + if (is_child) { + brush = brush.Clone(); + brush.Opacity = 0.75; + } + return brush; + } + } } } diff --git a/src/lib/config/protos/UI.proto b/src/lib/config/protos/UI.proto index 467b433ec..e2aece834 100644 --- a/src/lib/config/protos/UI.proto +++ b/src/lib/config/protos/UI.proto @@ -71,6 +71,10 @@ message LadderKnobs { optional bool use_algo_param_grid = 7 [default = true]; } +message OrderBookOptions { + optional bool animate_child_order_rows = 1 [default = false]; +} + message ShellPingOptions { optional bool shell_ping_is_enabled = 1 [default = true]; optional int32 workloop_milliseconds_interval = 2 [default = 5000]; From a29eaa2618bbfc8c294954e2ef4f0b68c2b4259b Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sat, 11 May 2019 12:25:38 -0500 Subject: [PATCH 69/85] EnumCaseStringConverter - cleanup code style/syntax --- .../EnumConverters/EnumCaseStringConverter.cs | 27 +++++-------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/src/client/spark/view/Converters/EnumConverters/EnumCaseStringConverter.cs b/src/client/spark/view/Converters/EnumConverters/EnumCaseStringConverter.cs index b267e2d86..696313310 100644 --- a/src/client/spark/view/Converters/EnumConverters/EnumCaseStringConverter.cs +++ b/src/client/spark/view/Converters/EnumConverters/EnumCaseStringConverter.cs @@ -1,28 +1,15 @@ using System; -using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using System.Windows.Data; -namespace mbts.ui.converters -{ - public class EnumCaseStringConverter : IValueConverter where T : struct - { +namespace mbts.ui.converters { + class EnumCaseStringConverter : IValueConverter where T : struct { public object Convert(object value, Type targetType, object parameter, System.Globalization.CultureInfo culture) { - T t = default(T); - if (value is T) - t = (T)value; - string raw_str = t.ToString(); - return raw_str.Substring(0, 1) + new String(raw_str.Skip(1).Select(Char.ToLower).ToLArray()); + string raw_str = (value is T t ? t : default).ToString(); + return raw_str.Substring(0, 1) + new string(raw_str.Skip(1).Select(char.ToLower).ToArray()); } - public object ConvertBack(object value, Type targetType, object parameter, System.Globalization.CultureInfo culture) { - T t = default(T); - string str = value as string; - if (str != null) - Enum.TryParse(str, true, out t); - return t; - } + public object ConvertBack(object value, Type targetType, object parameter, System.Globalization.CultureInfo culture) => + value is string str && Enum.TryParse(str, true, out T result) ? result : default; } -} +} \ No newline at end of file From 7bb077a74d7fa6ada6dc2282bd2576cc615ff9a3 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sat, 11 May 2019 12:27:42 -0500 Subject: [PATCH 70/85] remove ToLArray extension (it is not more efficient than Enumerable.ToArray()) --- src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs | 2 +- src/client/spark/Modules/Trading/TradingShellModule.cs | 6 +++--- src/client/spark/core/Collections/ObservantDictionary.cs | 2 +- .../DataGridEx/Filters/VM/DiscreteValuesFilterVM.cs | 5 +---- .../Collections/Extensions/EnumerableExtensions.cs | 5 +---- 5 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs b/src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs index 297cf49d9..8af2dcfdf 100644 --- a/src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs +++ b/src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs @@ -73,7 +73,7 @@ namespace mbts.spark { _has_working_orders = backing(nameof(has_working_orders), false); contract_picker_model = new ContractPickerModel(bus, false, null, new SecurityCriteria(new[] { new SecurityCriteriaClause() { - exchanges = EnumHelper.get_values_except(Venue.Bts).ToLArray() + exchanges = EnumHelper.get_values_except(Venue.Bts).ToArray() } })); contract_picker_model.cancel_action = _close_new_leg_popup; diff --git a/src/client/spark/Modules/Trading/TradingShellModule.cs b/src/client/spark/Modules/Trading/TradingShellModule.cs index 7408057bb..c5cf0da00 100644 --- a/src/client/spark/Modules/Trading/TradingShellModule.cs +++ b/src/client/spark/Modules/Trading/TradingShellModule.cs @@ -6,6 +6,7 @@ using mbts.ui; using System.Collections.Generic; using System.ComponentModel.Composition; using System.IO; +using System.Linq; using System.Threading.Tasks; using System.Windows.Media; using static mbts.ui.shell.ProtoBufSerializationHelper; @@ -52,9 +53,8 @@ namespace bts.shell { yield return _aswf; } - public Task create(Stream load_stream) { - return dispatch_to_factory(load_stream, window_factories().ToLArray()); - } + public Task create(Stream load_stream) => + dispatch_to_factory(load_stream, window_factories().ToArray()); public void Dispose() { foreach (var i in window_factories()) diff --git a/src/client/spark/core/Collections/ObservantDictionary.cs b/src/client/spark/core/Collections/ObservantDictionary.cs index cd05ae51d..b63029364 100644 --- a/src/client/spark/core/Collections/ObservantDictionary.cs +++ b/src/client/spark/core/Collections/ObservantDictionary.cs @@ -42,7 +42,7 @@ namespace mbts.collections foreach (TModel m in e.OldItems) _remove(m); break; case NotifyCollectionChangedAction.Reset: - var old_views = Values.ToLArray(); + var old_views = Values.ToArray(); Clear(); foreach (TValue v in old_views) ValueDestroyed?.Invoke(v); foreach (TModel m in _models) _add(m); diff --git a/src/client/spark/view/Controls/DataGridEx/Filters/VM/DiscreteValuesFilterVM.cs b/src/client/spark/view/Controls/DataGridEx/Filters/VM/DiscreteValuesFilterVM.cs index fbc33e8e5..5eb943e76 100644 --- a/src/client/spark/view/Controls/DataGridEx/Filters/VM/DiscreteValuesFilterVM.cs +++ b/src/client/spark/view/Controls/DataGridEx/Filters/VM/DiscreteValuesFilterVM.cs @@ -3,17 +3,14 @@ using bts.collections; using bts.utils; using mbts.collections; using mbts.ui.shell; -using mbts.ui.utilities; using mbts.ui.utility; using System; -using System.Collections; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Collections.Specialized; using System.ComponentModel; using System.Linq; using System.Reactive.Linq; -using System.Windows.Data; namespace mbts.spark.filters { static class DiscreteValuesFilterVM { @@ -87,7 +84,7 @@ namespace mbts.spark.filters { public IReadOnlyCollection items { get; } public ObservableCollectionEx selected_items { get; } = new ObservableCollectionEx(); - public override TValue[] get_state() => selected_items.ToLArray(); + public override TValue[] get_state() => selected_items.ToArray(); protected override void _clear() { _destroy_snapshot(); selected_items.Clear(); diff --git a/src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs b/src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs index 6f7d59c46..d2f85f273 100644 --- a/src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs +++ b/src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs @@ -30,11 +30,8 @@ namespace System.Collections.Generic { public static IEnumerable OfTypeSafe(this IEnumerable source) => source == null ? Enumerable.Empty() : source.OfType(); - public static T[] ToLArray(this IEnumerable enumerable) => - enumerable.ToList().ToArray(); - public static TAccumulate AggregateWhile(this IEnumerable source, TAccumulate seed, Func aggregate, Func predicate) => - AggregateWhile(source, seed, aggregate, predicate, (curr, el, next) => curr); + AggregateWhile(source, seed, aggregate, predicate, (curr, el, next) => curr); public static TAccumulate AggregateWhile(this IEnumerable source, TAccumulate seed, Func aggregate, Func predicate, TAccumulate early_break_rv) => AggregateWhile(source, seed, aggregate, predicate, (curr, el, next) => early_break_rv); From c7899a1dae869e17622f0b7da0931c2d4123d512 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sat, 11 May 2019 12:47:07 -0500 Subject: [PATCH 71/85] SPARK-2434 OrderBook - security column filter popup is empty if shell is started before server has published all securities --- .../Controls/DataGridEx/Filters/VM/DiscreteValuesFilterVM.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/spark/view/Controls/DataGridEx/Filters/VM/DiscreteValuesFilterVM.cs b/src/client/spark/view/Controls/DataGridEx/Filters/VM/DiscreteValuesFilterVM.cs index 5eb943e76..3658c5413 100644 --- a/src/client/spark/view/Controls/DataGridEx/Filters/VM/DiscreteValuesFilterVM.cs +++ b/src/client/spark/view/Controls/DataGridEx/Filters/VM/DiscreteValuesFilterVM.cs @@ -144,10 +144,10 @@ namespace mbts.spark.filters { ) : this(get_row_val, get_state, load_state, prop_name, equality_comparer, comparer) { // _items_sub = items.OnDispatcher().Subscribe(update => { - foreach (var v in update.added) - _items_set.Add(v); foreach (var v in update.removed.Except(_selected_items_set)) _items_set.Remove(v); + foreach (var v in update.added) + _items_set.Add(v); _unsorted_items.sync_with(_items_set); }); } From 870e7a9ef961106546363eb82e2488c11d3e6c0a Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sun, 12 May 2019 12:10:50 -0500 Subject: [PATCH 72/85] DIYGrid - added mutable InactiveSelectionOpacity prop (was just hardcoded at 0.2) --- .../spark/view/Controls/DIYGrid/DIYGrid.cs | 31 ++++++++++++++++--- src/client/spark_api/utils/MathUtils.cs | 4 +++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/client/spark/view/Controls/DIYGrid/DIYGrid.cs b/src/client/spark/view/Controls/DIYGrid/DIYGrid.cs index e1077760c..b3ff38bfa 100644 --- a/src/client/spark/view/Controls/DIYGrid/DIYGrid.cs +++ b/src/client/spark/view/Controls/DIYGrid/DIYGrid.cs @@ -242,6 +242,7 @@ namespace mbts.ui.controls { public static readonly SolidColorBrush DefaultSelectionHighlight = new SolidColorBrush(Colors.MediumSlateBlue) { Opacity = 0.4 }; public static readonly Pen DefaultSelectionStroke = new Pen(Brushes.MediumSlateBlue, 1); readonly NotifyCollectionChangedEventHandler _sort_descriptions_collection_changed_cb; + double _inactive_selection_opacity = 0.2; DataContextHooks _data_context_hooks; public static readonly DependencyProperty SelectionHighlightProperty = @@ -249,7 +250,7 @@ namespace mbts.ui.controls { nameof(SelectionHighlight), (o, v) => { o.Resources[ActiveSelectionHighlightKey] = v; - o.Resources[InactiveSelectionHighlightKey] = new SolidColorBrush(v.Color) { Opacity = 0.2 }.frozen(); + o._reset_inactive_selection_brush(); o.Resources[SelectionStrokeKey] = new Pen(o.SelectionHighlight.Color.to_solid_brush().frozen(), 1).frozen(); foreach (var col in o.Columns.empty_if_null().OfType()) col.on_selection_fill(v); @@ -322,14 +323,24 @@ namespace mbts.ui.controls { } } })); - Resources[InactiveSelectionHighlightKey] = new SolidColorBrush(SelectionHighlight.Color) { Opacity = 0.2 }; Resources[ActiveSelectionHighlightKey] = SelectionHighlight; + _reset_inactive_selection_brush(); _data_context_hooks = new DataContextHooks(this, DataContext); DataContextChanged += OnDataContextChanged; } public event DIYGridSortingEventHandler Sorting; + public double InactiveSelectionOpacity { + get => _inactive_selection_opacity; + set { + if (value != _inactive_selection_opacity && value.is_within_range(0.1, 0.9)) { + _inactive_selection_opacity = value; + _reset_inactive_selection_brush(); + } + } + } + public ItemContainerGenerator RowContainerGenerator => _row_presenter.ItemContainerGenerator; @@ -376,6 +387,18 @@ namespace mbts.ui.controls { void OnDataContextChanged(object sender, DependencyPropertyChangedEventArgs e) => Disposable.replace(ref _data_context_hooks, new DataContextHooks(this, e.NewValue)); + void _reset_inactive_selection_brush() { + try { + if (TryFindResource(ActiveSelectionHighlightKey) is SolidColorBrush brush) { + var clone = brush.Clone(); + if (_inactive_selection_opacity < clone.Opacity) { + clone.Opacity = _inactive_selection_opacity; + } + Resources[InactiveSelectionHighlightKey] = clone; + } + } catch { } + } + #region Column Handling private void invalidate_column_layout(object sender = null, EventArgs args = null) { var geometry = GetColumnGeometry().ToList(); @@ -591,7 +614,7 @@ namespace mbts.ui.controls { _row_presenter.ItemContainerStyle = _row_style; } - public Style RowStyle { get => GetValue(RowStyleProperty) as Style; set => SetValue(StyleProperty, value); } + public Style RowStyle { get => GetValue(RowStyleProperty) as Style; set => SetValue(RowStyleProperty, value); } #endregion public static readonly DependencyProperty CurrentRowProperty = DependencyProperty.Register( @@ -915,4 +938,4 @@ namespace mbts.ui.controls { } }); } -} \ No newline at end of file +} diff --git a/src/client/spark_api/utils/MathUtils.cs b/src/client/spark_api/utils/MathUtils.cs index 34b80caf2..7c2ebfaf4 100644 --- a/src/client/spark_api/utils/MathUtils.cs +++ b/src/client/spark_api/utils/MathUtils.cs @@ -53,6 +53,10 @@ namespace bts.utils { for (int i = 1; i < num2; i++) if ((num1 * i) % num2 == 0) return i * num1; return num1 * num2; } + + public static bool is_within_range(this double value, double min, double max) => + value >= min && value <= max; + public static int within_range(int min, int max, int value) { if (value < min) return min; if (value > max) return max; From 768567dfc700405d85d9408f90ad21539b768134 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sun, 12 May 2019 12:14:38 -0500 Subject: [PATCH 73/85] HighPerformanceRow - added BackgroundProperty --- .../DIYGrid/Row/HighPerformanceRow.cs | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/client/spark/view/Controls/DIYGrid/Row/HighPerformanceRow.cs b/src/client/spark/view/Controls/DIYGrid/Row/HighPerformanceRow.cs index 77b375065..eb1213a7c 100644 --- a/src/client/spark/view/Controls/DIYGrid/Row/HighPerformanceRow.cs +++ b/src/client/spark/view/Controls/DIYGrid/Row/HighPerformanceRow.cs @@ -11,7 +11,22 @@ using System.Windows.Media; namespace mbts.ui.controls { partial class DIYGrid { - public class HighPerformanceRow : FrameworkElement where TRowVM : class, IRowVM { + public class HighPerformanceRowBase : FrameworkElement { + protected Brush cached_background; + + public static readonly DependencyProperty BackgroundProperty = + WPFHelper.create_dp( + nameof(Background), + FrameworkPropertyMetadataOptions.AffectsRender, + (o, v) => o.cached_background = v); + + public Brush Background { + get => GetValue(BackgroundProperty) as Brush; + set => SetValue(BackgroundProperty, value); + } + } + + public class HighPerformanceRow : HighPerformanceRowBase where TRowVM : class, IRowVM { double _row_height; ObservableCollection _columns; List _column_layout = new List(); @@ -85,6 +100,9 @@ namespace mbts.ui.controls { _element_cells.VisualAtIndex(index); protected override void OnRender(DrawingContext dc) { + if (cached_background != null) { + dc.DrawRectangle(cached_background, null, new Rect(0, 0, ActualWidth, ActualHeight)); + } if (_row != null) { foreach (var cell in _column_layout) { if (!cell.column.is_collapsed && cell.column is IRenderColumn rc) From ea45c5c694666cf5531c933eaeee4bfec100aad6 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sun, 12 May 2019 12:18:01 -0500 Subject: [PATCH 74/85] OrderBook - added config for status color opacity when row is selected --- .../OrderBook/View/OrderBookGridView.xaml | 62 +------------------ .../OrderBook/View/OrderBookGridView.xaml.cs | 50 ++++++++++++++- src/lib/config/protos/UI.proto | 1 + 3 files changed, 51 insertions(+), 62 deletions(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/View/OrderBookGridView.xaml b/src/client/spark/Modules/Trading/OrderBook/View/OrderBookGridView.xaml index 94f2043e5..b9c7a1421 100644 --- a/src/client/spark/Modules/Trading/OrderBook/View/OrderBookGridView.xaml +++ b/src/client/spark/Modules/Trading/OrderBook/View/OrderBookGridView.xaml @@ -4,13 +4,12 @@ x:Name="_grid" xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" - xmlns:order="clr-namespace:bts.api.orderman;assembly=spark_api" xmlns:sp="clr-namespace:mbts.spark" - xmlns:u="clr-namespace:mbts.ui" xmlns:c="clr-namespace:mbts.ui.controls" xmlns:usp="clr-namespace:mbts.ui.spark" xmlns:uspcv="clr-namespace:mbts.ui.spark.converters" xmlns:sys="clr-namespace:System;assembly=mscorlib" + InactiveSelectionOpacity="0.3" > @@ -109,63 +108,4 @@ - - - - - - - - \ No newline at end of file diff --git a/src/client/spark/Modules/Trading/OrderBook/View/OrderBookGridView.xaml.cs b/src/client/spark/Modules/Trading/OrderBook/View/OrderBookGridView.xaml.cs index d5e489c78..e718abc21 100644 --- a/src/client/spark/Modules/Trading/OrderBook/View/OrderBookGridView.xaml.cs +++ b/src/client/spark/Modules/Trading/OrderBook/View/OrderBookGridView.xaml.cs @@ -1,4 +1,11 @@ +using bts.api.orderman; +using mbts.ui; using mbts.ui.controls; +using mbts.ui.converters; +using System.Windows; +using System.Windows.Controls; +using System.Windows.Data; +using System.Windows.Media; namespace mbts.spark { internal partial class OrderBookGridView : DIYGrid { @@ -45,8 +52,49 @@ namespace mbts.spark { RootAlgoType = 1 << 29 } - public OrderBookGridView() => + public OrderBookGridView() { InitializeComponent(); + var row_template_factory = new FrameworkElementFactory(typeof(OrderBookRowView)); + row_template_factory.SetValue(OrderBookRowView.BackgroundProperty, RowBackgroundHelper.row_background_binding); + RowTemplate = new DataTemplate { VisualTree = row_template_factory }; + } + + static class RowBackgroundHelper { + public static readonly BindingBase row_background_binding; + + static RowBackgroundHelper() { + var bg_binding = new MultiBinding(); + bg_binding.Bindings.Add(new Binding(nameof(OrderBookRowVM.status))); + bg_binding.Bindings.Add(new Binding(nameof(ListBoxItem.IsSelected)) { RelativeSource = new RelativeSource(RelativeSourceMode.FindAncestor, typeof(ListBoxItem), 1) }); + double overlay_opacity = config.Configuration.get().status_color_opacity_when_selected; + if (overlay_opacity > 0.1 && overlay_opacity < 0.9) { + bg_binding.Converter = ValueConverter.create((status, is_selected) => { + var color = _status_color(status); + return color.HasValue ? new SolidColorBrush(color.Value) { Opacity = is_selected ? overlay_opacity : 1.0 } : null; + }); + } else { + bg_binding.Converter = ValueConverter.create((status, is_selected) => { + if (is_selected) + return null; + var color = _status_color(status); + return color.HasValue ? new SolidColorBrush(color.Value) : null; + }); + } + row_background_binding = bg_binding; + } + + static Color? _status_color(OrderStatus status) { + switch (status) { + case OrderStatus.Canceled: return BTSColors.Orders.canceled_order_background; + case OrderStatus.Completed: return BTSColors.Orders.completed_order_background; + case OrderStatus.Disconnected: + case OrderStatus.RejectedBySpark: + case OrderStatus.Error: + return BTSColors.Orders.error_order_background; + default: return null; + } + } + } } class OrderBookRowView : DIYGrid.HighPerformanceRow { diff --git a/src/lib/config/protos/UI.proto b/src/lib/config/protos/UI.proto index e2aece834..ceaea14db 100644 --- a/src/lib/config/protos/UI.proto +++ b/src/lib/config/protos/UI.proto @@ -73,6 +73,7 @@ message LadderKnobs { message OrderBookOptions { optional bool animate_child_order_rows = 1 [default = false]; + optional double status_color_opacity_when_selected = 2 [default = 0.35]; } message ShellPingOptions { From 2b51bda3e79eba393984f0c71871ff9e4f81d720 Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Mon, 13 May 2019 11:27:12 -0400 Subject: [PATCH 75/85] Update system test scripts --- src/client/spark_test/System/LimitsTests.cs | 2 +- ...der_tests.sh => spreader_simple_sanity.sh} | 0 .../system_test_scripts/test_clip_limits.sh | 21 ++++++++++- .../system_test_scripts/test_join_algo.sh | 35 ++++++++++++------- .../test_position_limits.sh | 21 ++++++++++- 5 files changed, 64 insertions(+), 15 deletions(-) rename src/client/spark_test/system_test_scripts/{spreader_tests.sh => spreader_simple_sanity.sh} (100%) diff --git a/src/client/spark_test/System/LimitsTests.cs b/src/client/spark_test/System/LimitsTests.cs index 0eb96bf24..0a3357561 100644 --- a/src/client/spark_test/System/LimitsTests.cs +++ b/src/client/spark_test/System/LimitsTests.cs @@ -104,7 +104,7 @@ namespace bts.tests var all_securities = await all_futures(test_symbol); List security_list = all_securities.ToList(); - Assert.IsTrue(all_securities.Count() >= 2); + Assert.IsTrue(all_securities.Count() >= 2, "At least two securities are needed"); SecurityImmutable first_sec = security_list[0]; SecurityImmutable second_sec = security_list[1]; diff --git a/src/client/spark_test/system_test_scripts/spreader_tests.sh b/src/client/spark_test/system_test_scripts/spreader_simple_sanity.sh similarity index 100% rename from src/client/spark_test/system_test_scripts/spreader_tests.sh rename to src/client/spark_test/system_test_scripts/spreader_simple_sanity.sh diff --git a/src/client/spark_test/system_test_scripts/test_clip_limits.sh b/src/client/spark_test/system_test_scripts/test_clip_limits.sh index e90607b04..8928e476d 100644 --- a/src/client/spark_test/system_test_scripts/test_clip_limits.sh +++ b/src/client/spark_test/system_test_scripts/test_clip_limits.sh @@ -2,6 +2,24 @@ rm -r ~/sparkdata/data/databases/ > /dev/null + +mv ~/sparkdata/conf/spark.conf ~/sparkdata/conf/spark.conf.bak +cat << EOF > ~/sparkdata/conf/spark.conf + ExchangeSim { + value { + merge_with_md:true + } + } + Logging { + value { + console { default_level: "Error" } + syslog { default_level: "Error" } + file { default_level: "Error" } + } +} +EOF + + nohup ./sparkd run & pid=$! echo "sparkd pid: " $pid @@ -11,4 +29,5 @@ sleep 120 dotnet sparkcmd.dll test test_clip_limits -kill $pid \ No newline at end of file +kill $pid +mv ~/sparkdata/conf/spark.conf.bak ~/sparkdata/conf/spark.conf \ No newline at end of file diff --git a/src/client/spark_test/system_test_scripts/test_join_algo.sh b/src/client/spark_test/system_test_scripts/test_join_algo.sh index a9dd79390..775ee0b05 100644 --- a/src/client/spark_test/system_test_scripts/test_join_algo.sh +++ b/src/client/spark_test/system_test_scripts/test_join_algo.sh @@ -55,7 +55,7 @@ say_verbose() { is_merge_with_md_on() { eval $invocation - + if [ $(./sparkd configcheck ExchangeSim | grep merge_with_md: | cut -d " " -f 2) == "false" ]; then return 0 else @@ -65,14 +65,14 @@ is_merge_with_md_on() { set_merge_with_md_off() { eval $invocation - + if [ -f $config_file_path ]; then say_verbose "config file exists ... removing" rm $config_file_path fi - + echo "$merge_with_md_key" >> $config_file_path && test_config_file_created=true - + return 0 } @@ -81,29 +81,37 @@ set_merge_with_md_off() { # class_name - $2 get_tests_in_class() { eval $invocation - + local return_var=$1 local class_name=$2 - local arr=($(dotnet sparkcmd.dll test --system --list | perl -ne '/^( *)/; $i = length $1; $j && $i >= $j and print, next; $j = 0; /'"$class_name"'/ and $j = $i + 1, print' | tail -n +2)) + local arr=($(dotnet sparkcmd.dll test --type System --list | perl -ne '/^( *)/; $i = length $1; $j && $i >= $j and print, next; $j = 0; /'"$class_name"'/ and $j = $i + 1, print' | tail -n +2)) eval $return_var='("${arr[@]}")' - + return 0 } +# Start sparkd +nohup ./sparkd run & +pid=$! +echo "sparkd pid: " $pid +echo "sleeping for 120 seconds..." +sleep 120 + + # args # class_name - $1 run_bts_test() { eval $invocation - + class_name=$1 - + local test_names_tuple=() get_tests_in_class test_names_tuple $class_name - for test_name in ${test_names_tuple[@]}; do + for test_name in ${test_names_tuple[@]}; do say_verbose "running test with name: $test_name ..."; dotnet sparkcmd.dll test $test_name done - + return 0 } @@ -127,7 +135,7 @@ if is_merge_with_md_on; then say_err "Please investigate manually" exit 1 fi -else +else say_verbose "merge with md is already off" fi @@ -137,4 +145,7 @@ if [ "$test_config_file_created" = true ]; then rm $config_file_path fi +# Kill sparkd +kill $pid + say "Test has finished." diff --git a/src/client/spark_test/system_test_scripts/test_position_limits.sh b/src/client/spark_test/system_test_scripts/test_position_limits.sh index 06efd0c0c..7a688b7fe 100644 --- a/src/client/spark_test/system_test_scripts/test_position_limits.sh +++ b/src/client/spark_test/system_test_scripts/test_position_limits.sh @@ -2,6 +2,24 @@ rm -r ~/sparkdata/data/databases/ > /dev/null + +mv ~/sparkdata/conf/spark.conf ~/sparkdata/conf/spark.conf.bak +cat << EOF > ~/sparkdata/conf/spark.conf + ExchangeSim { + value { + merge_with_md:true + } + } + Logging { + value { + console { default_level: "Error" } + syslog { default_level: "Error" } + file { default_level: "Error" } + } +} +EOF + + nohup ./sparkd run & pid=$! echo "sparkd pid: " $pid @@ -10,4 +28,5 @@ echo "sleeping for 120 seconds..." sleep 120 dotnet sparkcmd.dll test test_position_limits -kill $pid \ No newline at end of file +kill $pid +mv ~/sparkdata/conf/spark.conf.bak ~/sparkdata/conf/spark.conf \ No newline at end of file From f6cf0f28ea4d629a9a2faad145c86e15571d3739 Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Mon, 13 May 2019 13:31:02 -0400 Subject: [PATCH 76/85] Move test scripts in spark_test.csproj --- src/client/spark/Spark.csproj | 4 +--- src/client/spark_test/spark_test.csproj | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/client/spark/Spark.csproj b/src/client/spark/Spark.csproj index cc84e43f4..3f23b08d7 100644 --- a/src/client/spark/Spark.csproj +++ b/src/client/spark/Spark.csproj @@ -91,9 +91,7 @@ MSBuild:Compile - - - + diff --git a/src/client/spark_test/spark_test.csproj b/src/client/spark_test/spark_test.csproj index 8a6c94761..308bc8928 100644 --- a/src/client/spark_test/spark_test.csproj +++ b/src/client/spark_test/spark_test.csproj @@ -11,4 +11,8 @@ + + + + From 21ee7c6af5fa0e20642488e32115dc70fde9d336 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 13 May 2019 17:30:49 -0400 Subject: [PATCH 77/85] 2.0.0 Release notes. --- doc/user/content/release_notes/_index.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/user/content/release_notes/_index.md b/doc/user/content/release_notes/_index.md index f37e8d038..f60201f44 100644 --- a/doc/user/content/release_notes/_index.md +++ b/doc/user/content/release_notes/_index.md @@ -4,6 +4,22 @@ weight = 3 sort_by = "weight" insert_anchor_links = "right" +++ +## 2.0.0 +### User Visible Changes + - Support for attaching and detaching algos to existing orders + - Support for display and update of all parameters in running algos + - Streamlining of order badge decorations in ladder + - New order search/filtering algorithm for order book + - Simpler representation of complex algo chains in order book and mini order book + - Visual and usability improvements to mini order book + - Streamlining of algo parameter dialog in ladder + - New spreader queue holder behaviour + - Visual improvements to the order book + - New server side API +### Internal changes + - Exchange simulator no longer requires daily sequence number reset + - Native orders can now buffer updates/cancels when disconected or pending first acknowledgment + ## 1.6.2 ### User Visible Changes ### Exchange simulator From 8575d92373ee7bcb8db06fe524c8593fc807c8ff Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Mon, 13 May 2019 17:52:46 -0400 Subject: [PATCH 78/85] once the lean's child has crossed, changin lean's price has no effect on the child --- .../orders/algo/algo_orders/LeanAlgo.cpp | 16 +++++++++++++--- .../orderman/orders/algo/algo_orders/LeanAlgo.h | 3 ++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp index a7f0a870b..6a86c4671 100644 --- a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp +++ b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp @@ -28,6 +28,7 @@ struct LeanAlgo::PImpl { Book curr_book; Price out_price = Price(os::nan()); Quantity out_qty = Quantity(0); + Price cross_price = Price(os::nan()); }; LeanAlgo::LeanAlgo() : me(new PImpl()) {} @@ -79,7 +80,8 @@ void LeanAlgo::do_update(const api::orderman::Update &update) { return; } Update child_update; - child_update.set_price(update.price()); + Price price_to_pass = _get_price_to_pass(Price(update.price())); + child_update.set_price(price_to_pass.value()); child_update.set_quantity(update.quantity()); if (update.has_custom()) { auto &lean_update = update.custom().lean(); @@ -195,6 +197,13 @@ LeanAlgo::MarketChange LeanAlgo::_market_change() const{ else return MarketChange::SIZE_CHANGE; } +Price LeanAlgo::_get_price_to_pass (Price price) const{ + // if it has crossed once, then the + // child order is always passed that price + if(!os::is_nan(me->cross_price.value())) + return me->cross_price; + return price; +} // actions //////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////// void LeanAlgo::_analyze_and_act() { @@ -298,8 +307,9 @@ void LeanAlgo::_udpate_out_price_qty(Price price, Quantity qty){ } bool LeanAlgo::_modify_price_price_qty(Price price, Quantity qty){ if(me->out_price.value()!=price.value() || - me->out_qty.value()!=qty.value()){ - _udpate_out_price_qty(price,qty); + me->out_qty.value()!=qty.value()){ + _udpate_out_price_qty(price,qty); + me->cross_price = price; Update child_update; child_update.set_price(price.value()), child_update.set_quantity(qty.value()); diff --git a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.h b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.h index 56bda12d3..09aea41ba 100644 --- a/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.h +++ b/src/lib/orderman/orders/algo/algo_orders/LeanAlgo.h @@ -34,7 +34,8 @@ class LeanAlgo : public IAlgoOrder { api::Side _opp_side() const; bool _within_chase_band(Price price) const; Price _chase_limit() const; - MarketChange _market_change() const; + MarketChange _market_change() const; + Price _get_price_to_pass (Price price) const; void _analyze_and_act(); bool _try_cross(Price); void _go_furthest_possible(); From 39efacace4025c3c666c3bf6138767b727e3e2b2 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Tue, 14 May 2019 08:56:56 -0400 Subject: [PATCH 79/85] Updated release notes --- doc/user/content/release_notes/_index.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/user/content/release_notes/_index.md b/doc/user/content/release_notes/_index.md index f60201f44..d48a43747 100644 --- a/doc/user/content/release_notes/_index.md +++ b/doc/user/content/release_notes/_index.md @@ -16,9 +16,14 @@ insert_anchor_links = "right" - New spreader queue holder behaviour - Visual improvements to the order book - New server side API + - Improved admin UI performance + - Optimized PIQ rendering ### Internal changes - Exchange simulator no longer requires daily sequence number reset + - Improved MD resiliance to corrupted shared memory - Native orders can now buffer updates/cancels when disconected or pending first acknowledgment + - Improved settings manager tool output + - Added diagnostics in the event MD memory corruption is detected ## 1.6.2 ### User Visible Changes From 1cad5b6b390d8a61ece4b764ab435bf56a4c108f Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Tue, 14 May 2019 09:56:47 -0400 Subject: [PATCH 80/85] Updated release notes. --- doc/user/content/release_notes/_index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/user/content/release_notes/_index.md b/doc/user/content/release_notes/_index.md index d48a43747..8ae152ca8 100644 --- a/doc/user/content/release_notes/_index.md +++ b/doc/user/content/release_notes/_index.md @@ -24,6 +24,7 @@ insert_anchor_links = "right" - Native orders can now buffer updates/cancels when disconected or pending first acknowledgment - Improved settings manager tool output - Added diagnostics in the event MD memory corruption is detected + - Added systems tests for join,lean, iceberg and spreader orders ## 1.6.2 ### User Visible Changes From 6c51ff83d5920926f746426bfdda220ed310aabd Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Tue, 14 May 2019 10:01:39 -0400 Subject: [PATCH 81/85] Release notes again. --- doc/user/content/release_notes/_index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/user/content/release_notes/_index.md b/doc/user/content/release_notes/_index.md index 8ae152ca8..e7be97cc7 100644 --- a/doc/user/content/release_notes/_index.md +++ b/doc/user/content/release_notes/_index.md @@ -25,6 +25,7 @@ insert_anchor_links = "right" - Improved settings manager tool output - Added diagnostics in the event MD memory corruption is detected - Added systems tests for join,lean, iceberg and spreader orders + - Made exchange order recovery more resiliant in the presence of database loss ## 1.6.2 ### User Visible Changes From 316be12b716381e178b828a55fe2e60ee21462e5 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Tue, 14 May 2019 11:46:27 -0400 Subject: [PATCH 82/85] Return order key in API create. This broke in 1944. --- src/lib/orderman/server/OrderServiceServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/orderman/server/OrderServiceServer.cpp b/src/lib/orderman/server/OrderServiceServer.cpp index a8f4a8acf..af49ce401 100644 --- a/src/lib/orderman/server/OrderServiceServer.cpp +++ b/src/lib/orderman/server/OrderServiceServer.cpp @@ -361,7 +361,7 @@ void OrderServiceServer::on_order_op_result( bts_assert(it!=me->in_flight_order_ops.end()); auto &responder = it->second; api::orderman::Response r; - r.set_order_key(0); + r.set_order_key(op_result.ok.value()); r.set_error_code(op_result.result.error); r.set_msg(op_result.result.error_msg); responder(r); From d9221e1053b59d7d38f42e69e5df360e2b041a89 Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Tue, 14 May 2019 12:54:01 -0400 Subject: [PATCH 83/85] fix system test scripts --- .../spreader_simple_sanity.sh | 15 +++++++++++---- .../system_test_scripts/test_clip_limits.sh | 16 +++++++++++----- .../system_test_scripts/test_position_limits.sh | 15 +++++++++++---- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/client/spark_test/system_test_scripts/spreader_simple_sanity.sh b/src/client/spark_test/system_test_scripts/spreader_simple_sanity.sh index 01f2a03de..66e563fb2 100644 --- a/src/client/spark_test/system_test_scripts/spreader_simple_sanity.sh +++ b/src/client/spark_test/system_test_scripts/spreader_simple_sanity.sh @@ -1,9 +1,16 @@ #!/bin/bash -rm -r ~/sparkdata/data/databases/ > /dev/null +if [ -d ~/sparkdata/ ] +then + sparkdata_dir=~/sparkdata +else + sparkdata_dir='/sparkdata' +fi -mv ~/sparkdata/conf/spark.conf ~/sparkdata/conf/spark.conf.bak -cat << EOF > ~/sparkdata/conf/spark.conf +rm -r $sparkdata_dir/data/databases/ > /dev/null + +mv $sparkdata_dir/conf/spark.conf $sparkdata_dir/conf/spark.conf.bak +cat << EOF > $sparkdata_dir/conf/spark.conf ExchangeSim { value { merge_with_md:false @@ -29,4 +36,4 @@ sleep 120 dotnet sparkcmd.dll test spreader_simple_sanity kill $pid -mv ~/sparkdata/conf/spark.conf.bak ~/sparkdata/conf/spark.conf \ No newline at end of file +mv $sparkdata_dir/conf/spark.conf.bak $sparkdata_dir/conf/spark.conf \ No newline at end of file diff --git a/src/client/spark_test/system_test_scripts/test_clip_limits.sh b/src/client/spark_test/system_test_scripts/test_clip_limits.sh index 8928e476d..18aae3d8c 100644 --- a/src/client/spark_test/system_test_scripts/test_clip_limits.sh +++ b/src/client/spark_test/system_test_scripts/test_clip_limits.sh @@ -1,10 +1,17 @@ #!/bin/bash -rm -r ~/sparkdata/data/databases/ > /dev/null +if [ -d ~/sparkdata/ ] +then + sparkdata_dir=~/sparkdata +else + sparkdata_dir='/sparkdata' +fi + +rm -r $sparkdata_dir/data/databases/ > /dev/null -mv ~/sparkdata/conf/spark.conf ~/sparkdata/conf/spark.conf.bak -cat << EOF > ~/sparkdata/conf/spark.conf +mv $sparkdata_dir/conf/spark.conf $sparkdata_dir/conf/spark.conf.bak +cat << EOF > $sparkdata_dir/conf/spark.conf ExchangeSim { value { merge_with_md:true @@ -19,7 +26,6 @@ cat << EOF > ~/sparkdata/conf/spark.conf } EOF - nohup ./sparkd run & pid=$! echo "sparkd pid: " $pid @@ -30,4 +36,4 @@ sleep 120 dotnet sparkcmd.dll test test_clip_limits kill $pid -mv ~/sparkdata/conf/spark.conf.bak ~/sparkdata/conf/spark.conf \ No newline at end of file +mv $sparkdata_dir/conf/spark.conf.bak $sparkdata_dir/conf/spark.conf \ No newline at end of file diff --git a/src/client/spark_test/system_test_scripts/test_position_limits.sh b/src/client/spark_test/system_test_scripts/test_position_limits.sh index 7a688b7fe..1ee160075 100644 --- a/src/client/spark_test/system_test_scripts/test_position_limits.sh +++ b/src/client/spark_test/system_test_scripts/test_position_limits.sh @@ -1,10 +1,17 @@ #!/bin/bash -rm -r ~/sparkdata/data/databases/ > /dev/null +if [ -d ~/sparkdata/ ] +then + sparkdata_dir=~/sparkdata +else + sparkdata_dir='/sparkdata' +fi + +rm -r $sparkdata_dir/data/databases/ > /dev/null -mv ~/sparkdata/conf/spark.conf ~/sparkdata/conf/spark.conf.bak -cat << EOF > ~/sparkdata/conf/spark.conf +mv $sparkdata_dir/conf/spark.conf $sparkdata_dir/conf/spark.conf.bak +cat << EOF > $sparkdata_dir/conf/spark.conf ExchangeSim { value { merge_with_md:true @@ -29,4 +36,4 @@ sleep 120 dotnet sparkcmd.dll test test_position_limits kill $pid -mv ~/sparkdata/conf/spark.conf.bak ~/sparkdata/conf/spark.conf \ No newline at end of file +mv $sparkdata_dir/conf/spark.conf.bak $sparkdata_dir/conf/spark.conf \ No newline at end of file From 3144032ed6b1a82c4aff9da25c8dffcbc9e34684 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 14 May 2019 13:46:03 -0400 Subject: [PATCH 84/85] Fix dictionary key with ShellLatency (add user) --- src/server/sparkd/support/ShellLatencyServer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server/sparkd/support/ShellLatencyServer.cpp b/src/server/sparkd/support/ShellLatencyServer.cpp index ccdb3e4ca..eb5657b87 100644 --- a/src/server/sparkd/support/ShellLatencyServer.cpp +++ b/src/server/sparkd/support/ShellLatencyServer.cpp @@ -36,7 +36,7 @@ void ShellLatencyServer::report( reply(bts::api::VoidMessage()); // look up or create the metric - auto key = request.host_name() + "-" + request.version(); + auto key = request.host_name() + "-" + request.version() + "-" + request.user(); auto iter = _metrics.find(key); std::shared_ptr metric; // report at least 1ms because the metric will not report anything if it's 0 @@ -44,8 +44,8 @@ void ShellLatencyServer::report( if (iter == _metrics.end()) { // create metric auto metric = - std::make_shared("spark_shell_latency", - std::vector