Compare commits

..
Author SHA1 Message Date
John Colvin 5dd1b5089a build: Add build-spark-prod-installer.Jenkinsfile 2023-07-23 23:10:56 -04:00
John Colvin 923e0f5502 fix build: Fix jenkins build script
Clean it up a bit in the process
2023-07-21 14:27:39 -04:00
John Colvin 6d45e0d494 temp 2023-07-20 00:36:40 -04:00
John Colvin a29ed2b7bd temp 2023-07-20 00:35:47 -04:00
John Colvin 3fe447dc67 Don't wipe workspace 2023-07-20 00:15:43 -04:00
John Colvin f78cb33088 wipe workspace 2023-07-19 23:45:29 -04:00
John Colvin 9c09be6ced temp 2023-07-19 23:39:31 -04:00
John Colvin a220e86e95 temp 2023-07-19 23:33:23 -04:00
John Colvin 5a2c15d4be build: Add Jenkinsfile for Spark 2023-07-19 14:55:47 -04:00
20 changed files with 354 additions and 500 deletions

No files matched your search

@@ -0,0 +1,82 @@
pipeline {
agent {
label 'os=windows && ram=8gb && installed=python3'
}
parameters {
choice(name: 'JOB', choices: ['build-spark-prod'])
string(name: 'BUILD_NUMBER', defaultValue: '', description: "The build number for which to make the installer (runs new build if blank).")
string(name: 'RELEASE_ID', defaultValue: 'latest', description: "The release id (aka version tag). For ex, 'v1.1.1'")
booleanParam(name: 'CLEAN', defaultValue: false, description: "Whether or not to perform a 'clean' build. If true, 'BUILD_NUMBER' must be blank.")
string(name: 'CONFIG_SERVER', defaultValue: "http://<server_ip>:<config_port>", description: "Default server url for the installer.")
}
stages {
stage('Validate Params') {
steps {
script {
if ("${params.BUILD_NUMBER}" != "") {
if (params.CLEAN) {
error "BUILD_NUMBER must be left blank for CLEAN builds"
}
if (!("${params.BUILD_NUMBER}" =~ "^[1-9][0-9]*\$")) {
error "BUILD_NUMBER must be positive non-zero integer (or left blank)"
}
}
if ("${params.RELEASE_ID}" == "") {
error "RELEASE_ID must not be blank"
}
}
}
}
stage('Build') {
when {
expression { params.CLEAN || params.BUILD_NUMBER == '' }
}
steps {
script {
build(
job: params.JOB,
parameters: [
booleanParam(name: 'CLEAN', value: params.CLEAN),
booleanParam(name: 'CLEAN_REPO', value: false),
booleanParam(name: 'CLEAN_VENV', value: params.CLEAN)
])
}
}
}
stage('Build Installer') {
steps {
script {
src_dir='installer_src'
out_dir='installer'
out_name="spark-${params.RELEASE_ID}"
echo 'Cleaning files leftover from previous builds'
fileOperations([
fileDeleteOperation(includes: 'publish.zip'),
folderDeleteOperation(folderPath: src_dir),
folderDeleteOperation(folderPath: out_dir)])
if (params.BUILD_NUMBER == '') {
copyArtifacts(projectName: params.JOB, selector: lastSuccessfulBuild());
} else {
copyArtifacts(projectName: params.JOB, selector: specific("${params.BUILD_NUMBER}"));
}
fileOperations([
fileUnZipOperation(filePath: 'publish.zip', targetLocation: src_dir),
fileCreateOperation(fileName: "${src_dir}\\spark.version", fileContent: params.RELEASE_ID)])
innosetup_exe="C:\\Program Files (x86)\\Inno Setup 6\\ISCC.exe"
innosetup_args = String.join(
" ",
"spark-prod-client.iss",
"/DMySourceDir=${src_dir}",
"/DMyAppVersion=${params.RELEASE_ID}",
"/DMyConfigServer=\"${params.CONFIG_SERVER}\"",
"/O${out_dir}",
"/F${out_name}");
bat "@call \"${innosetup_exe}\" ${innosetup_args}"
}
archiveArtifacts artifacts: "${out_dir}\\${out_name}.exe", followSymlinks: false, onlyIfSuccessful: true
}
}
}
}
+72
View File
@@ -0,0 +1,72 @@
pipeline {
agent {
label 'os=windows && ram=8gb && installed=vs2022 && installed=python3'
}
parameters {
booleanParam(name: 'CLEAN', defaultValue: false, description: 'Perform a clean build')
booleanParam(name: 'CLEAN_VENV', defaultValue: false, description: 'Create the python venv from scratch (delete existing)')
gitParameter(
name: 'TAG',
type: 'PT_BRANCH_TAG',
defaultValue: 'origin/spark_crypto.prod',
listSize: '0',
sortMode: 'ASCENDING_SMART')
}
stages {
stage('Build') {
steps {
checkout([
$class: 'GitSCM',
branches: [[name: "${params.TAG}"]],
extensions: [
[
$class: 'SubmoduleOption',
disableSubmodules: false,
recursiveSubmodules: true,
trackingSubmodules: true,
parentCredentials: true,
],
[
$class: 'CloneOption',
],
[
$class: 'PruneStaleBranch'
],
[
$class: 'PruneStaleTag'
],
[
$class: 'WipeWorkspace'
]
],
userRemoteConfigs: [[url: "https://john_colvin:ATBBLTcLcmD4ysvbC8D9j5sXhpMAF4312F8F@bitbucket.org/bluetradesys/spark.git"]]
])
script {
// Python venv
if ("${params.CLEAN_VENV}") {
echo "Cleaning python virtual environment..."
bat "@call rmdir /s /q .venv > nul 2>&1"
}
bat '@call tools\\ensure_venv.cmd'
// Main build
if ("${params.CLEAN}") {
echo "Cleaning build"
bat "@call rmdir /s /q .venv > nul 2>&1"
}
bat '@call tools\\jenkins_build_spark.cmd --target publish'
}
}
}
stage('Archive') {
steps {
dir ('build\\Release\\bin\\net7.0-windows\\win-x64') {
bat """
"C:\\Program Files\\7-Zip\\7z.exe" a -tzip ..\\..\\..\\..\\..\\spark.zip publish
"""
}
archiveArtifacts artifacts: 'spark.zip', followSymlinks: false, onlyIfSuccessful: true
}
}
}
}
@@ -160,8 +160,8 @@ namespace mbts.spark {
public static IEnumerable<bts.api.SecurityTuple> get_vs_sectups(
int vs_id, ThreadedBlackboard bb) {
var vs_usage_list = bb.read_state(SparkTags.VolSurfaceUsageTag);
return (vs_usage_list?.get_sectups(vs_id)).empty_if_null();
VolSurfaceUsageListImmutable vs_usage_list = bb.read_state(SparkTags.VolSurfaceUsageTag);
return vs_usage_list.get_sectups(vs_id);
}
@@ -242,4 +242,4 @@ namespace mbts.spark {
}
#endregion
}
}
}
@@ -63,9 +63,9 @@ namespace mbts.spark {
// find the symbol vs vs_id, get the committed vs, see if it is
// owned by the current user, and only then publish it
HashSet<string> syms = new HashSet<string>();
var usage = _bb.read_state(SparkTags.VolSurfaceUsageTag);
VolSurfaceUsageListImmutable usage = _bb.read_state(SparkTags.VolSurfaceUsageTag);
string current_user = _bb.read_state(SparkTags.CurrentUser).name;
foreach (var u in (usage?.UsageList).empty_if_null()) {
foreach (var u in usage.UsageList) {
VolSurfaceInput vs_input =
_bb.read_state(SparkTags.VolSurfaceCommittedInputTag(u.VsId));
if (vs_input != null) {
@@ -135,4 +135,4 @@ namespace mbts.spark {
_bb.data_callback -= _on_data;
}
}
}
}
@@ -22,7 +22,7 @@ namespace mbts.spark {
// ii. IntepGreeks for each vs
public class VolSurfaceInterpGreeksMultiplexer : IDisposable {
private readonly ThreadedBlackboard _bb;
private readonly ITag<VolSurfaceUsageListImmutable?> _in_bb_vs_usage_tag;
private readonly ITag<VolSurfaceUsageListImmutable> _in_bb_vs_usage_tag;
private IDictionary<int, ITag<VSTheos>> _vs_theo_tags =
new Dictionary<int, ITag<VSTheos>>();
private VolSurfaceUsageListImmutable _vs_usage_map =
@@ -48,8 +48,8 @@ namespace mbts.spark {
}
private void _analyze_usages() {
_vs_usage_map = _bb.read_state(_in_bb_vs_usage_tag) ??
new VolSurfaceUsageList().immutable();
_vs_usage_map = _bb.read_state(_in_bb_vs_usage_tag,
new VolSurfaceUsageListImmutable(new VolSurfaceUsageList()));
bts.Log.Gen.info($"VolSurfaceInterpGreeksMultiplexer._analyze_usages " +
$"count: {_vs_usage_map.UsageList.Count()}");
foreach (int vs_id in _vs_usage_map.vss()) {
@@ -40,8 +40,10 @@ namespace mbts.analytics {
/// last computed min/max driver for each exp
private IDictionary<VolSurfaceExpiration, Tuple<double, double>>
_computed_driver_range_map = new Dictionary<VolSurfaceExpiration, Tuple<double, double>>();
private readonly IEnumerable<double> _default_abs_moves = config.Configuration.get<bts.config.VolSurface>().default_abs_moves;
private readonly IEnumerable<double> _default_abs_moves =
new double[] { 300, -200, -125, -100, -75, -50, -25, 0, 25, 50, 75, 100, 125, 200, 300 };
private readonly double _cushion_percentage = 25.0;
public event
Action<IDictionary<VolSurfaceExpiration, IEnumerable<Theos>>, double, double> on_theos =
(o, vt, tt) => { };
@@ -50,7 +52,7 @@ namespace mbts.analytics {
(o, vt, tt) => { };
public VolSurfaceInterpGreeksProducer(int vs_id) {
Log.Gen.info($"VolSurfaceInterpGreeksProducer.cons vs_id: {vs_id}, _default_abs_moves: {string.Join(", ", _default_abs_moves)}");
Log.Gen.info($"VolSurfaceInterpGreeksProducer.cons vs_id: {vs_id}");
_vs_id = vs_id;
}
@@ -1281,7 +1281,7 @@ namespace mbts.spark {
if (!_settings.piq_column.show)
return;
int origin_index = LadderRender.index_of(_price_list, _mkt_break_lines.origin, _price_comparer);
LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, _price_list, _vm.price_vs_piq, _settings.row_height, _spec, _settings.piq_column, _fmt_mkt_quantity);
LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, origin_index, _price_list, _vm.price_vs_piq, _settings.row_height, _spec, _settings.piq_column, _fmt_mkt_quantity);
}
void _refresh_sizes() {
@@ -368,32 +368,30 @@ namespace mbts.spark {
public delegate void RenderCell<T>(DrawingContext dc, int row, T data_item);
static void _render_cells<T>(DrawingVisual dv, double row_height, IEnumerable<(double price, T value)> cells, LadderColumnSettings settings, IList<double> prices, RegionalTickFactors spec, RenderCell<T> render_cell) {
static void _render_cells<T>(DrawingVisual dv, double row_height, IEnumerable<(double price, T value)> data, LadderColumnSettings settings, IList<double> prices, RegionalTickFactors spec, RenderCell<T> render_cell) {
using (var dc = dv.RenderOpen())
using (_draw_with_clip(dc, settings, row_height, prices.Count)) {
if (prices.is_null_or_empty() || cells.is_null_or_empty()) {
if (data.is_null_or_empty())
return;
}
var min_price = prices[prices.Count - 1];
var max_price = prices[0];
Log.assert(min_price.is_real() && max_price.is_real() && min_price <= max_price);
var indexed_cells = cells
.Where(item => item.price.is_real() && item.price.is_within_range(min_price, max_price))
.Select(row => {
checked {
return (row.price, row.value, (int)spec.tickdiff(max_price, row.price));
}
});
foreach (var (price, value, idx) in indexed_cells) {
var check_price = spec.tickdown(max_price, idx);
Log.assert(spec.tick_equals(price, check_price));
Log.assert(idx >= 0 && idx < prices.Count);
render_cell(dc, idx, value);
int curr_index = 0;
double curr_price = prices[curr_index];
foreach (var (price, value) in data) {
if (double.IsNaN(price))
continue;
// prices are in descending order, so get diff from curr_price - price
var tick_diff = spec.tickdiff(curr_price, price) ?? 0;
Log.assert(
tick_diff > (int.MaxValue - curr_index),
$"Unable to index ladder row for price {price}: tick count exceeds int.MaxValue");
curr_index += (int)tick_diff;
curr_price = price;
if (curr_index >= 0 && curr_index < prices.Count) {
render_cell(dc, curr_index, value);
} else break;
}
}
}
internal static void render_last(DrawingVisual dv, GlyphContext glyph_context, NativePriceSize last, int index, double row_height, LadderColumnSettings settings, Func<uint, string> fmt_value) {
if (fmt_value is null)
fmt_value = FormattingDelegates.default_string<uint>();
@@ -407,18 +405,38 @@ namespace mbts.spark {
internal static void render_piqs(
DrawingVisual dv,
GlyphContext glyph_context,
int origin_index,
IList<double> prices,
IEnumerable<(double price, uint piq)> pvp,
double row_height,
RegionalTickFactors spec,
LadderColumnSettings settings,
Func<uint, string> fmt_value
) {
fmt_value ??= FormattingDelegates.default_string<uint>();
var fg = settings.foreground.rgb_brush();
_render_cells(dv, row_height, pvp, settings, prices, spec, (dc, row, piq) =>
dc.DrawGlyphRun(
fg, format_text(fmt_value(piq), row, row_height, settings, glyph_context)));
Func<uint, string> fmt_value) {
//
using (var dc = dv.RenderOpen()) {
using (_draw_with_clip(dc, settings, row_height, prices.Count)) {
if (!prices.within_range(origin_index) || pvp.is_null_or_empty())
return;
if (fmt_value is null)
fmt_value = FormattingDelegates.default_string<uint>();
var foreground = settings.foreground.rgb_brush();
int curr_index = origin_index;
double curr_price = prices[curr_index];
foreach (var entry in pvp) {
double price = entry.price;
// prices are in descending order, so get diff from curr_price - price
var tick_diff = spec.tickdiff(curr_price, price).assert_value();
checked {
curr_index += (int)tick_diff;
}
curr_price = price;
if (curr_index >= 0 && curr_index < prices.Count) {
var gr = format_text(fmt_value(entry.piq), curr_index, row_height, settings, glyph_context);
dc.DrawGlyphRun(foreground, gr);
} else break;
}
}
}
}
public static void render_mkt_break(DrawingVisual dv, int index, double row_height, uint color) =>
@@ -59,9 +59,7 @@ namespace bts.shell.manual_fills {
nameof(quantity_display),
() => {
var sec_meta = security.HasValue ? _sec_meta_store.get(security.Value) : null;
return sec_meta?.HasQuantityDisplay ?? false ?
sec_meta.Value.QuantityDisplay :
Security.Defaults.quantity_display_settingsi;
return sec_meta?.QuantityDisplay ?? Security.Defaults.quantity_display_settingsi;
});
// TODO (prob): remove conversion_data. Replace get_qty_display_divisor delegates w/
// TickDisplaySettingsImmutable for qty, obtained via ISecurityMetadataStore.
@@ -293,10 +293,7 @@ namespace bts.shell.pnl {
return _quantity_display_backing.Value;
}
if (security.HasValue) {
var sec_meta = _sec_meta_store.get(security.Value);
if (sec_meta?.HasQuantityDisplay ?? false) {
_quantity_display_backing = sec_meta.Value.QuantityDisplay;
}
_quantity_display_backing = _sec_meta_store.get(security.Value)?.QuantityDisplay;
}
return _quantity_display_backing ??= Security.Defaults.quantity_display_settingsi;
}
@@ -121,7 +121,7 @@ namespace bts.shell {
void _on_orders(TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> update) {
// must update notification_service_data_cache first, some of the predicate/triggers (esp. ToastNotifications) will use the notification_service_data_cache to lookup order-history
_data_cache.update(update);
_data_cache.update(update.as_iincremental());
foreach (var o in update.added.Concat(update.changed))
_order_notifications.process_item(o);
}
@@ -1,157 +1,89 @@
using bts.api;
using bts.api.core;
using bts.api.orderman;
using mbts.spark;
using mbts.utils;
using System;
using System.Collections.Generic;
using System.Linq;
namespace bts.shell {
// Any predicate-sound/notifications that need to reference previous data (i.e. notifications based on change/time, or state changes) should use the INotificationServiceDataCache.
// The INotificationService updates these caches in its IObservable.OnNext callbacks, just before calling all of the predicate-trigger methods.
// Therfore, data referenced through this interface is guarenteed to be up-to-date, and thread safe. It also reduces performance overhead by avoiding redundant bookeeping.
// Note: last_order_reset_time can be used by sound/toast-notification sources to avoid sending false-positive notifications for old orders (i.e. all the orders that arrive in the first update on startup/server restart)
// --------------------------------------------------------------------------------------------------------------------------------------------
// IMPOARTANT!!! The current implementation of this interface is NOT INTENDED FOR GENERAL USE - order_history is NOT THREAD-SAFE
interface INotificationServiceDataCache {
bool is_first_order_update { get; }
IReadOnlyDictionary<uint, List<TimeStampedData<OrderSecurity>>> order_history { get; }
}
static class NotificationServiceDataCacheExtensions {
public static OrderSecurity? get_previous_order_state(this INotificationServiceDataCache notification_service_data_cache, uint key) {
if (notification_service_data_cache.order_history.TryGetValue(key, out List<TimeStampedData<OrderSecurity>> l)) {
if (l.Count > 1) {
return l[l.Count - 2].data;
}
}
return null;
}
}
struct TimeStampedData<T> {
public readonly T data;
public readonly DateTime time_stamp;
public TimeStampedData(T data, DateTime time_stamp) {
this.data = data;
this.time_stamp = time_stamp;
}
}
partial class NotificationService : INotificationService, IDisposable {
class DataCache : INotificationServiceDataCache {
const int _ORDER_HISTORY_BUFFER_SIZE = 10;
const int _TABLE_HISTORY_BUFFER_SIZE = 5;
readonly Dictionary<uint, OrderSecurity> _order_cache = new Dictionary<uint, OrderSecurity>();
readonly Dictionary<uint, List<TimeStampedData<OrderSecurity>>> _order_history = new Dictionary<uint, List<TimeStampedData<OrderSecurity>>>();
readonly List<(TableUpdate<(SecurityImmutable?, OrderRecordImmutable), uint>, DateTime)> _table_history = new(_TABLE_HISTORY_BUFFER_SIZE);
readonly object _lock = new();
public IReadOnlyDictionary<uint, List<TimeStampedData<OrderSecurity>>> order_history => _order_history;
public bool is_first_order_update { get; private set; }
internal void update(TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> table) {
// First, try processing the update per usual (incrementally). If it throws,
// then log the exception and reset the state.
DateTime now = DateTime.Now;
lock (_lock) {
_circular_buffer_add(_table_history, (table, now), _TABLE_HISTORY_BUFFER_SIZE);
try {
impl();
return;
} catch (Exception ex) {
var table_history_desc = _fmt_order_table_history_safe();
Log.Ui.error(
$"Error processing order update in NotificationService: {ex}.\n" +
$"***Last table updates below***\n" +
table_history_desc);
}
// This is the 'escape' path. It really shouldn't throw.
reset();
}
void impl() {
if (table.state.Count == 0) {
_order_cache.Clear();
}
is_first_order_update = _order_cache.Count == 0 && table.state.Count > 0;
foreach (var order in table.removed) {
var k = table.key(order);
_order_cache.Remove(k);
_order_history.Remove(k);
}
foreach (var ch in table.changes) {
var k = table.key(ch.New);
_order_cache[k] = ch.New;
_order_history[k].Add(new(ch.New, now));
}
foreach (var order in table.added) {
var k = table.key(order);
_order_cache[k] = order;
var hist = _order_history.GetOrAdd(k, _ => new());
_circular_buffer_add(hist, new(order, now), _ORDER_HISTORY_BUFFER_SIZE);
}
}
void reset() {
_order_cache.Clear();
_order_history.Clear();
_table_history.Clear();
foreach (var (k, order) in table.state) {
_order_cache[k] = order;
_order_history.GetOrAdd(k, _ => new()).Add(new(order, now));
}
is_first_order_update = true;
}
}
static void _circular_buffer_add<T>(List<T> list, T item, int buffer_size) {
Log.assert(buffer_size > 0);
if (list.Count > buffer_size - 1) {
// Remove oldest items (trim front)
list.RemoveRange(
0, list.Count - (buffer_size - 1));
}
list.Add(item);
}
string _fmt_order_table_history_safe() {
return _fmt_table_history_safe(_table_history, fmt_order_tup);
static string fmt_order_tup((SecurityImmutable?, OrderRecordImmutable) order_tup) {
var (sec, order) = order_tup;
var order_desc = OrderUtils.short_description(order, sec.spec_or_default());
var sec_desc = sec?.ToString() ?? ("sid=" + order.security_key().ToString());
return $"{sec_desc} {order_desc}";
}
}
static string _fmt_table_history_safe<T, TKey>(
List<(TableUpdate<T, TKey>, DateTime)> table_history, Func<T, string> fmt_value) where TKey : IEquatable<TKey> {
var table_history_desc = new System.Text.StringBuilder();
for (int i = 0; i < table_history.Count; i++) {
var (table, time_stamp) = table_history[i];
table_history_desc.AppendLine(
$"Update - {table_history.Count - i}, at {time_stamp:yyyy-MM-dd hh:mm:ss.ffffff}");
table_history_desc.AppendLine(_fmt_table_safe(table, fmt_value));
}
return table_history_desc.ToString();
}
static string _fmt_table_safe<T, TKey>(
TableUpdate<T, TKey> table, Func<T, string> fmt_value) where TKey : IEquatable<TKey> {
try {
return table.description(fmt_value);
} catch (Exception ex) {
return $"Exception while formatting TableUpdate: ${ex}";
}
}
}
}
}
using bts.api;
using bts.api.orderman;
using mbts.spark;
using mbts.utils;
using System;
using System.Collections.Generic;
namespace bts.shell {
// Any predicate-sound/notifications that need to reference previous data (i.e. notifications based on change/time, or state changes) should use the INotificationServiceDataCache.
// The INotificationService updates these caches in its IObservable.OnNext callbacks, just before calling all of the predicate-trigger methods.
// Therfore, data referenced through this interface is guarenteed to be up-to-date, and thread safe. It also reduces performance overhead by avoiding redundant bookeeping.
// Note: last_order_reset_time can be used by sound/toast-notification sources to avoid sending false-positive notifications for old orders (i.e. all the orders that arrive in the first update on startup/server restart)
// --------------------------------------------------------------------------------------------------------------------------------------------
// IMPOARTANT!!! The current implementation of this interface is NOT INTENDED FOR GENERAL USE - order_history is NOT THREAD-SAFE
interface INotificationServiceDataCache {
DateTime last_order_reset_time { get; }
bool is_first_order_update { get; }
IReadOnlyDictionary<uint, List<TimeStampedData<OrderSecurity>>> order_history { get; }
}
static class NotificationServiceDataCacheExtensions {
public static OrderSecurity? get_previous_order_state(this INotificationServiceDataCache notification_service_data_cache, uint key) {
if (notification_service_data_cache.order_history.TryGetValue(key, out List<TimeStampedData<OrderSecurity>> l)) {
if (l.Count > 1) {
return l[l.Count - 2].data;
}
}
return null;
}
}
struct TimeStampedData<T> {
public readonly T data;
public readonly DateTime time_stamp;
public TimeStampedData(T data, DateTime time_stamp) {
this.data = data;
this.time_stamp = time_stamp;
}
}
partial class NotificationService : INotificationService, IDisposable {
class DataCache : INotificationServiceDataCache {
const int _buffer_size = 10;
readonly Dictionary<uint, OrderSecurity> _order_cache = new Dictionary<uint, OrderSecurity>();
readonly Dictionary<uint, List<TimeStampedData<OrderSecurity>>> _order_history = new Dictionary<uint, List<TimeStampedData<OrderSecurity>>>();
public IReadOnlyDictionary<uint, List<TimeStampedData<OrderSecurity>>> order_history => _order_history;
public DateTime last_order_reset_time { get; private set; }
public bool is_first_order_update { get; private set; }
internal void update(IIncrementalUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> data) {
DateTime now = DateTime.Now;
if (data.removed.Count == _order_cache.Count) {
last_order_reset_time = now;
_order_cache.Clear();
}
is_first_order_update = _order_cache.Count == 0 && data.added.Count > 0;
foreach (var kv in data.removed) {
if (data.added.ContainsKey(kv.Key)) {
_order_cache[kv.Key] = kv.Value;
_order_history[kv.Key].Add(new TimeStampedData<OrderSecurity>(kv.Value, now));
} else {
_order_cache.Remove(kv.Key);
// TODO: maybe these shouldn't be removed?
// if we add delta/time predicates that need to refer to order history, we might need to keep them around
// although, if the server is telling us to remove them, perhaps we shouldn't consider such data.
_order_history.Remove(kv.Key);
}
}
foreach (var kv in data.added) {
if (data.removed.ContainsKey(kv.Key))
continue;
_order_cache.Add(kv.Key, kv.Value);
_order_history.AddOrUpdate(kv.Key,
k => new List<TimeStampedData<OrderSecurity>> { new TimeStampedData<OrderSecurity>(kv.Value, now) },
(k, l) => {
l.Add(new TimeStampedData<OrderSecurity>(kv.Value, now));
if (l.Count >= 10)
l.RemoveAt(0);
});
}
}
}
}
}
@@ -90,8 +90,7 @@ namespace mbts {
}
var sec = order.security;
var sec_meta = sec.HasValue ? _sec_meta_store.get(sec.Value) : null;
var qty_display = sec_meta?.HasQuantityDisplay ?? false ?
sec_meta.Value.QuantityDisplay :
var qty_display = sec_meta?.QuantityDisplay ??
Security.Defaults.quantity_display_settingsi;
return ToastNotifications.order_completed(order, qty_display);
}
+2 -2
View File
@@ -266,8 +266,8 @@ namespace mbts.spark {
// List of symbol to volsurface mapping.
// Publisher: VolSurfaceBlackboardManager
public static ITag<VolSurfaceUsageListImmutable?> VolSurfaceUsageTag { get; } =
_mk_tag<VolSurfaceUsageListImmutable?>();
public static ITag<VolSurfaceUsageListImmutable> VolSurfaceUsageTag { get; } =
_mk_tag<VolSurfaceUsageListImmutable>();
// The AW2 machinary can compute and publish the theos
// right back locally(vs the server doing that) if configured that way.
@@ -10,8 +10,8 @@
</ResourceDictionary.MergedDictionaries>
<Style x:Key="LogViewer.TextBox" TargetType="TextBox">
<Setter Property="Background" Value="{u:BTSColor UI.TextBox.default_background}" />
<Setter Property="Foreground" Value="{u:BTSColor UI.TextBox.default_foreground}" />
<Setter Property="Background" Value="{StaticResource TextBox.BackgroundBrush}" />
<Setter Property="Foreground" Value="{StaticResource TextBox.ForegroundBrush}" />
<Setter Property="BorderThickness" Value="0" />
</Style>
@@ -1,22 +0,0 @@
VolSurface {
value {
default_abs_moves: -600
default_abs_moves: -300
default_abs_moves: -200
default_abs_moves: -125
default_abs_moves: -100
default_abs_moves: -75
default_abs_moves: -50
default_abs_moves: -25
default_abs_moves: 0
default_abs_moves: 25
default_abs_moves: 50
default_abs_moves: 75
default_abs_moves: 100
default_abs_moves: 125
default_abs_moves: 200
default_abs_moves: 300
default_abs_moves: 600
}
}
@@ -1,6 +0,0 @@
syntax = 'proto2';
package bts.config;
message VolSurface {
repeated double default_abs_moves = 1;
}
@@ -300,9 +300,6 @@ extern "C" void init_execution_tester_plugin(bts::SparkServices &srv) {
extern "C" void init_task_eye_plugin(bts::SparkServices &srv) {
DynamicOrderDescriptor tasking_eye_algo;
tasking_eye_algo.set_dynamic_order_type("Tasking Eye");
// if this is set to true, it waits for the existance of Signal File
*tasking_eye_algo.add_param_desc() = param_bool("Wait For Signal");
*tasking_eye_algo.add_param_desc() = param_string("Signal File");
*tasking_eye_algo.add_param_desc() = param_string("Account");
*tasking_eye_algo.add_param_desc() = param_double("Edge");
*tasking_eye_algo.add_param_desc() = param_double("Shot Size");
@@ -23,17 +23,10 @@
#include "lib/utils/bts_assert.h"
#include <bits/stdint-uintn.h>
#include <boost/algorithm/string.hpp>
#include <chrono>
#include <elf.h>
#include <filesystem>
#include <google/protobuf/timestamp.pb.h>
#include <google/protobuf/util/time_util.h>
#include <iostream>
#include <optional>
#include <sstream>
#include <string>
#include <thread>
#include <tuple>
#include <unordered_map>
using namespace bts::api::orderman;
@@ -76,11 +69,6 @@ double eps = 0.0000001;
bool eq(double d1, double d2) { return std::abs(d1 - d2) < eps; }
bool greater_then_eq(double d1, double d2) { return eq(d1, d2) || d2 > d1; }
void sleep_log() {
return;
std::this_thread::sleep_for(std::chrono::seconds(5));
}
// Returns the number of shots that are within the Shots Window
size_t n_recent_shots(
const std::vector<TEAShot> &shots, uint64_t window_nanos) {
@@ -112,91 +100,7 @@ bool in_flight_orders(const std::vector<TEAShot> &shots) {
return false;
}
// returns the securities that match the criteria from the 2nd - 8th arg
std::set<SecurityKey> match_securities(
const std::vector<api::Security const *> &secs,
api::Venue venue,
const std::string &symbol,
uint32_t exp_month,
uint32_t exp_day,
uint32_t exp_yr,
std::tuple<double, double> strike_range,
api::PutOrCall cp) {
std::set<SecurityKey> rv = {};
double strike_min = std::get<0>(strike_range);
double strike_max = std::get<1>(strike_range);
for (const auto &s : secs) {
if (s->exchange() == venue && s->has_option() && s->symbol() == symbol) {
const auto &oc = s->option();
if (oc.call_put() == cp && oc.strike() >= strike_min &&
oc.strike() <= strike_max && oc.expiration_year() == exp_yr &&
oc.expiration_month() == exp_month &&
oc.expiration_day() == exp_day)
rv.insert(SecurityKey(s->id()));
}
}
return rv;
}
// Parses the sec expressed as text in the secs string.
// the format is symbol:exp:call/put:strike range, ex
// BTC:08/14/2023:Call:25000-30000
std::set<SecurityKey> parse_securities_from_range(
const std::string &secs,
const std::vector<api::Security const *> &sec_vec) {
LOG(OrderMan, Info,
"TaskingEyeAlgo::parse_securities_from_range secs: {} sec_vec: {}", secs,
sec_vec.size());
sleep_log();
if (secs.empty())
return {};
auto parse_exp = [](const std::string &exp_str) {
LOG(OrderMan, Info,
"TaskingEyeAlgo::parse_securities_from_range parse_exp exp: {}",
exp_str);
std::vector<std::string> exp_parts;
boost::split(exp_parts, exp_str, boost::is_any_of("/"));
LOG(OrderMan, Info,
"TaskingEyeAlgo::parse_securities_from_range exp_parts: {}",
exp_parts.size());
uint32_t mm = static_cast<std::uint32_t>(std::stoul(exp_parts[0]));
uint32_t dd = static_cast<std::uint32_t>(std::stoul(exp_parts[1]));
uint32_t yy = static_cast<std::uint32_t>(std::stoul(exp_parts[2]));
LOG(OrderMan, Info,
"TaskingEyeAlgo::parse_securities_from_range {} mm: {} dd: {} yy: {}",
mm, dd, yy);
return std::tuple<uint32_t, uint32_t, uint32_t>(mm, dd, yy);
};
auto parse_strike_range = [](const std::string &strike_range_str) {
std::vector<std::string> strike_parts;
boost::split(strike_parts, strike_range_str, boost::is_any_of("-"));
double k_min = std::stod(strike_parts[0]);
double k_max = std::stod(strike_parts[1]);
return std::tuple<double, double>(k_min, k_max);
};
std::vector<std::string> parts;
boost::split(parts, secs, boost::is_any_of(":"));
const std::string &symbol = parts[0];
const std::string &exp_str = parts[1];
const std::string &cp_str = parts[2];
const auto &strike_range_str = parts[3];
LOG(OrderMan, Info,
"TaskingEyeAlgo::parse_securities_from_range symbol: {} exp: {} cp: {} "
"stike_range: {}",
symbol, exp_str, cp_str, strike_range_str);
// debugging
sleep_log();
auto [mm, dd, yy] = parse_exp(exp_str);
auto strike_range = parse_strike_range(strike_range_str);
api::PutOrCall cp = (cp_str[0] == 'C' || cp_str[0] == 'c')
? api::PutOrCall::PUTORCALL_CALL
: api::PutOrCall::PUTORCALL_PUT;
return match_securities(
sec_vec, api::Venue::VENUE_DERIBIT, symbol, mm, dd, yy, strike_range,
cp);
}
std::set<SecurityKey> parse_securities_from_keys(const std::string &secs) {
std::set<SecurityKey> parse_securities(const std::string &secs) {
std::set<SecurityKey> rv = {};
std::vector<std::string> parts;
boost::split(parts, secs, boost::is_any_of(","));
@@ -209,34 +113,10 @@ std::set<SecurityKey> parse_securities_from_keys(const std::string &secs) {
return rv;
}
std::set<SecurityKey> parse_securities(
const std::string &secs,
const std::vector<api::Security const *> &sec_vec) {
bts_assert(!secs.empty());
LOG(OrderMan, Info, "TaskingEyeAlgo::parse_securities sec: {}", secs);
sleep_log();
std::set<SecurityKey> rv = {};
if (secs.find("/") != std::string::npos &&
secs.find(":") != std::string::npos) {
rv = parse_securities_from_range(secs, sec_vec);
} else {
rv = parse_securities_from_keys(secs);
}
std::stringstream ss;
for (auto k : rv) {
ss << k << " ";
}
LOG(OrderMan, Info, "TaskingEyeAlgo::parse_securities sec: {} rv: {}", secs,
ss.str());
return rv;
}
struct TaskingEyeAlgo ::PImpl {
IAlgoServices *service_ptr = nullptr;
analytics::ITheoCalculator const *theo_calc = nullptr;
TEAState state = TEAState::ACTIVE;
bool wait_for_signal = false;
std::string signal_file = "";
double edge = 100000;
double shot_size = 0;
double target_qty = 0;
@@ -270,62 +150,31 @@ TaskingEyeAlgo::~TaskingEyeAlgo() {}
OrderOpResult TaskingEyeAlgo::do_before_launch(
IBeforeLaunchAlgoServices &serv) {
auto is_data_valid = [&]() {
bool rv = greater_then_eq(0.1, me->shot_size) &&
greater_then_eq(1, me->target_qty) &&
greater_then_eq(1, me->max_shots_per_window) &&
greater_then_eq(1000000000, me->window_size_nanos) &&
me->side != api::SIDE_UNDEFINED;
bool signal_correct = !me->wait_for_signal || !me->signal_file.empty();
rv = rv && signal_correct;
return rv;
return greater_then_eq(0.1, me->shot_size) &&
greater_then_eq(1, me->target_qty) &&
greater_then_eq(1, me->max_shots_per_window) &&
greater_then_eq(1000000000, me->window_size_nanos) &&
me->side != api::SIDE_UNDEFINED;
};
LOG(OrderMan, Info,
"TaskingEyeAlgo::do_before_launch order_key: {} security: {} "
"definition: {}",
serv.order_key().value(), utils::pb::to_string(serv.security()),
utils::pb::to_string(serv.definition()));
sleep_log();
auto &def = serv.definition();
if (def.has_custom_def() && def.custom_def().has_dynamic()) {
const auto &d_params = def.custom_def().dynamic().order_params();
auto wait_for_signal = get_if<bool>(d_params, "Wait For Signal");
me->wait_for_signal =
wait_for_signal.has_value() ? wait_for_signal.value() : false;
auto signal_file = get_if<std::string>(d_params, "Signal File");
me->signal_file = signal_file.has_value() ? signal_file.value() : "";
me->edge = get_if<double>(d_params, "Edge").value();
LOG(OrderMan, Info, "TaskingEyeAlgo::do_before_launch edge: {}",
me->edge);
sleep_log();
me->shot_size = get_if<double>(d_params, "Shot Size").value();
LOG(OrderMan, Info, "TaskingEyeAlgo::do_before_launch shot_size: {}",
me->shot_size);
sleep_log();
me->target_qty = get_if<double>(d_params, "Target Quantity").value();
LOG(OrderMan, Info, "TaskingEyeAlgo::do_before_launch target_qty: {}",
me->target_qty);
sleep_log();
// TODO, make sure the side if right
me->side = def.side();
me->max_shots_per_window =
get_if<uint32_t>(d_params, "Max Shots Per Window").value();
LOG(OrderMan, Info,
"TaskingEyeAlgo::do_before_launch max_shots_per_window: {}",
me->max_shots_per_window);
sleep_log();
uint32_t window_size_secs =
get_if<uint32_t>(d_params, "Window Size (secs)").value();
LOG(OrderMan, Info,
"TaskingEyeAlgo::do_before_launch window_size_secs: {}",
window_size_secs);
sleep_log();
me->window_size_nanos = seconds_to_nanos(window_size_secs);
auto sec_str = get_if<std::string>(d_params, "Securities").value();
LOG(OrderMan, Info, "TaskingEyeAlgo::do_before_launch Securities: {}",
sec_str);
sleep_log();
auto parsed_secs = parse_securities(sec_str, serv.known_securities());
std::stringstream ss;
auto parsed_secs = parse_securities(sec_str);
std::vector<SecurityKey> final_secs;
if (parsed_secs.empty()) {
final_secs.push_back(SecurityKey{serv.security().id()});
@@ -337,21 +186,18 @@ OrderOpResult TaskingEyeAlgo::do_before_launch(
auto ct = get_tea_child_template(serv, sec_key, me->side);
me->sec_map.insert({sec_key, {ct}});
LOG(OrderMan, Info,
"TaskingEyeAlgo::do_before_launch order_key: {} "
"successfully "
"TaskingEyeAlgo::do_before_launch order_key: {} successfully "
"created ChildTemplate for sec_key: {}",
serv.order_key(), sec_key);
}
LOG(OrderMan, Info,
"TaskingEyeAlgo::do_before_launch algo constructed order_key: "
"{} "
"edge: {} shot_size: {} side: {} target_qty: {} "
"max_shots_per_window: {} window_size_nanos: {} n_secs: {} "
"wait_for_signal: {} signal_file: {}",
"TaskingEyeAlgo::do_before_launch algo constructed order_key: {} "
"edge: {} shot_size: {} is_buy: {} target_qty: {} "
"max_shots_per_window: {} window_size_nanos: {} n_secs: {}",
serv.order_key(), me->edge, me->shot_size,
bts::api::Side_Name(me->side), me->target_qty,
me->side == api::SIDE_BUY, me->target_qty,
me->max_shots_per_window, me->window_size_nanos,
me->sec_map.size(), me->wait_for_signal, me->signal_file);
me->sec_map.size());
return {};
}
}
@@ -367,8 +213,7 @@ void TaskingEyeAlgo::do_launch(IAlgoServices &serv) {
for (const auto &kv : me->sec_map) {
auto sec_key = kv.first;
LOG(OrderMan, Info,
"TaskingEyeAlgo::do_before_launch subscribing book for sec_key: "
"{}",
"TaskingEyeAlgo::do_before_launch subscribing book for sec_key: {}",
sec_key);
me->service_ptr->subscribe_book(sec_key);
}
@@ -470,9 +315,9 @@ void TaskingEyeAlgo::on_theos(
if (!sec_data.driver_sec_key.has_value() ||
sec_data.driver_sec_key->value() != driver.value()) {
LOG(OrderMan, Info,
"TaskingEyeAlgo::on_theos order_key: {} subscribing to driver for "
"sec_key: {} driver_key: {}",
me->service_ptr->order_key(), sec_key, driver.value());
"TaskingEyeAlgo::on_theos order_key: {} subscribing to driver "
"sec_key: {}",
me->service_ptr->order_key(), driver.value());
sec_data.driver_sec_key = SecurityKey(driver.value());
sub_to_driver(SecurityKey(driver.value()));
}
@@ -486,14 +331,8 @@ void TaskingEyeAlgo::on_position(PositionUpdate const &pu) {
}
void TaskingEyeAlgo::_evaluate_and_shoot() {
if (me->state != TEAState::ACTIVE) {
LOG(OrderMan, Warn,
"TaskingEyeAlgo::_evaluate_and_shoot order_key: {} state is not "
"active state: {} ",
me->service_ptr->order_key(), me->state);
if (me->state != TEAState::ACTIVE)
return;
}
auto max_shots_per_win_reached = [&]() {
return n_recent_shots(me->shots, me->window_size_nanos) >=
me->max_shots_per_window;
@@ -504,36 +343,12 @@ void TaskingEyeAlgo::_evaluate_and_shoot() {
return true;
return false;
};
auto signal_condition_met = [&]() {
if (me->wait_for_signal) {
return std::filesystem::exists(me->signal_file);
} else
return true;
};
if (!signal_condition_met()) {
// Log ?
LOG(OrderMan, Warn,
"TaskingEyeAlgo::_evaluate_and_shoot order_key: {} Waiting for "
"Signal",
me->service_ptr->order_key());
return;
}
for (auto &kv : me->sec_map) {
auto sec_key = kv.first;
if (max_shots_per_win_reached()) {
LOG(OrderMan, Warn,
"TaskingEyeAlgo::_evaluate_and_shoot order_key: {} "
"max_shots_per_win_reached",
me->service_ptr->order_key());
if (max_shots_per_win_reached())
return;
}
if (sec_has_active_order(sec_key)) {
LOG(OrderMan, Warn,
"TaskingEyeAlgo::_evaluate_and_shoot sec_key: {} has working "
"order, skipping",
sec_key.value());
if (sec_has_active_order(sec_key))
continue;
}
Price price;
Quantity qty;
if (_in_shooting_zone(sec_key, price, qty)) {
@@ -542,8 +357,7 @@ void TaskingEyeAlgo::_evaluate_and_shoot() {
auto ch_ord_key =
me->service_ptr->launch_child(*sec_data.order, price, qty);
LOG(OrderMan, Info,
"TakingEyeAlgo::_evaluate_and_shoot ord_key: {} child_ord_key: "
"{} "
"TakingEyeAlgo::_evaluate_and_shoot ord_key: {} child_ord_key: {} "
"price: {} qty: {}",
me->service_ptr->order_key(), ch_ord_key, price, qty.full_value());
me->shots.push_back(TEAShot{sec_key, ch_ord_key});
@@ -561,13 +375,8 @@ bool TaskingEyeAlgo::_in_shooting_zone(
TEASecData const &sec_data = it->second;
const md::PriceSize &driver_bid = sec_data.driver_book.bid().at(0);
const md::PriceSize &driver_ask = sec_data.driver_book.ask().at(0);
if (eq(driver_bid.size(), 0) || eq(driver_ask.size(), 0)) {
LOG(OrderMan, Warn,
"TaskingEyeAlgo::_in_shooting_zone no driver for sec_key: {} "
"driver_bid_qty: {} driver_ask_qty: {}",
sec_key, driver_bid.size(), driver_ask.size());
if (eq(driver_bid.size(), 0) || eq(driver_ask.size(), 0))
return false;
}
// finding weighted mean (TODO change it to mark price later)
double driver = (driver_bid.price() * driver_bid.size() +
driver_ask.price() * driver_ask.size()) /
@@ -577,55 +386,31 @@ bool TaskingEyeAlgo::_in_shooting_zone(
// TODO normalize theo
double theo = ptp.theo();
theo = theo / driver;
// auto analyze_buy =
auto analyze_buy = [&](const md::PriceSize &md_ask, double theo,
size_t level, Price &price, Quantity &qty) -> bool {
if (me->side == api::SIDE_BUY) {
double ee_buy_pr = theo - me->edge;
bool rv = false;
const md::PriceSize &md_ask = sec_data.sec_book.ask().at(0);
LOG(OrderMan, Diag,
"TaskingEyeAlgo::_in_shooting_zone sec_key: {} driver: {} theo: {} "
"ee_price: {} md: {}/{}",
sec_key, driver, theo, ee_buy_pr, md_ask.price(), md_ask.size());
if (greater_then_eq(1, md_ask.size()) &&
greater_then_eq(md_ask.price(), ee_buy_pr)) {
price = Price(md_ask.price());
qty = me->shot_size;
rv = true;
qty = std::min(static_cast<double>(md_ask.size()), me->shot_size);
return true;
}
LOG(OrderMan, Diag,
"TaskingEyeAlgo::_in_shooting_zone::analyze_buy sec_key: {} "
"theo: {} level: {}"
"ee_buy_pr: {} rv: {}",
sec_key, level, theo, ee_buy_pr, rv);
return rv;
};
auto analyze_sell = [&](const md::PriceSize &md_bid, double theo,
size_t level, Price &price, Quantity &qty) -> bool {
} else {
double ee_sell_pr = theo + me->edge;
bool rv = false;
const md::PriceSize &md_bid = sec_data.sec_book.bid().at(0);
LOG(OrderMan, Diag,
"TaskingEyeAlgo::_in_shooting_zone sec_key: {} driver: {} theo: {} "
"ee_price: {} md: {}/{}",
sec_key, driver, theo, ee_sell_pr, md_bid.price(), md_bid.size());
if (greater_then_eq(1, md_bid.size()) &&
greater_then_eq(ee_sell_pr, md_bid.price())) {
price = Price(md_bid.price());
qty = me->shot_size;
rv = true;
}
LOG(OrderMan, Diag,
"TaskingEyeAlgo::_in_shooting_zone::analyze_sell sec_key: {} theo:"
"{} level: {} ee_sell_pr: {} rv: {}",
sec_key, theo, level, ee_sell_pr, rv);
return rv;
};
if (me->side == api::SIDE_BUY) {
const auto &ask_book = sec_data.sec_book.ask();
for (int i = ask_book.size() - 1; i >= 0; i--) {
const auto &md_ask = ask_book.at(i);
if (analyze_buy(md_ask, theo, i, price, qty)) {
return true;
}
}
} else {
const auto &bid_book = sec_data.sec_book.bid();
for (int i = bid_book.size() - 1; i >= 0; i--) {
const md::PriceSize &md_bid = bid_book.at(i);
if (analyze_sell(md_bid, theo, i, price, qty)) {
return true;
}
qty = std::min(static_cast<double>(md_bid.size()), me->shot_size);
return true;
}
}
return false;
+7 -7
View File
@@ -1,13 +1,13 @@
@echo off
@setlocal
@REM Build script designed for our jenkins build job(s). It assumes the py
@REM venv already exists.
if not defined VENV_DIR (
set VENV_DIR=%~dp0..\.venv
)
set BUILD_SPARK_PY="%~dp0py\build_spark.py"
setlocal
set tools_dir=%~dp0
set venv_dir=%~dp0..\.venv\
call "C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\Tools\VsDevCmd.bat"
call "%venv_dir%Scripts\activate.bat"
call python "%tools_dir%py\build_spark.py" %*
call "%VENV_DIR%\Scripts\activate"
call python "%BUILD_SPARK_PY%" %*
EXIT /B %ERRORLEVEL%