diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs index 4baa71e0b..8b0103ced 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs @@ -19,6 +19,8 @@ using Newtonsoft.Json; using BCG; using System.Runtime.CompilerServices; using bts; +using Spark.Core; +using System.Numerics; namespace bts { @@ -91,13 +93,13 @@ namespace BCG { return BlueDataUtility.get_row_datetime_value(r, col_no, err_value); } - public static double? GetDouble(this IBlueRow r, int colNo) => - GetDouble(r[colNo]); + public static double? GetFloat64(this IBlueRow r, int colNo) => + GetFloat64(r[colNo]); - public static double? GetDouble(this IBlueRow r, string colName) => - GetDouble(r[colName]); + public static double? GetFloat64(this IBlueRow r, string colName) => + GetFloat64(r[colName]); - public static double? GetDouble(object obj) { + public static double? GetFloat64(object obj) { if (obj is double d) { return d; } @@ -125,6 +127,97 @@ namespace BCG { return null; } + public static int? GetInt32(this IBlueRow r, int colNo) => + GetInt32(r[colNo]); + + public static int? GetInt32(this IBlueRow r, string colName) => + GetInt32(r[colName]); + + public static int? GetInt32(object obj) { + checked { + if (obj is double d) { + return (int)d; + } + if (obj is int i32) { + return i32; + } + if (obj is long i64) { + return (int)i64; + } + if (obj is string s) { + return double.TryParse(s, out d) ? (int)d : null; + } + if (obj is uint u32) { + return (int)u32; + } + if (obj is ulong u64) { + return (int)u64; + } + if (obj is short i16) { + return i16; + } + if (obj is ushort u16) { + return u16; + } + } + return null; + } + + public static long? GetInt64(this IBlueRow r, int colNo) => + GetInt64(r[colNo]); + + public static long? GetInt64(this IBlueRow r, string colName) => + GetInt64(r[colName]); + + public static long? GetInt64(object obj) { + checked { + if (obj is double d) { + return (long)d; + } + if (obj is int i32) { + return i32; + } + if (obj is long i64) { + return i64; + } + if (obj is string s) { + return double.TryParse(s, out d) ? (long)d : null; + } + if (obj is uint u32) { + return u32; + } + if (obj is ulong u64) { + return (long)u64; + } + if (obj is short i16) { + return i16; + } + if (obj is ushort u16) { + return u16; + } + } + return null; + } + + public static DateTime? GetUnixEpochDateTime(this IBlueRow row, string colName) => + GetUnixEpochDateTime(row[colName]); + + public static DateTime? GetUnixEpochDateTime(this IBlueRow row, int colNo) => + GetUnixEpochDateTime(row[colNo]); + + public static DateTime? GetUnixEpochDateTime(object obj) { + var tsNanos = GetInt64(obj); + return tsNanos.HasValue ? DateTime.UnixEpoch.Add(new TimeSpan(tsNanos.Value / 100)) : null; + } + + public static DateTime? GetDateTime(object obj) { + var res = _get_datetime_value(obj, DateTime.MinValue); + if (res == DateTime.MinValue) { + return null; + } + return res; + } + public static T get_enum(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct { return BlueDataUtility.get_row_enum_value(r, col_name, err_value); } diff --git a/src/DataStreamSuite/SharedStreams/RowTemplates/AvgOrderLatencySchema.json b/src/DataStreamSuite/SharedStreams/RowTemplates/AvgOrderLatencySchema.json new file mode 100644 index 000000000..7e5ca4d3b --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/RowTemplates/AvgOrderLatencySchema.json @@ -0,0 +1,34 @@ +{ + "IsTimeSeries": true, + "Columns": [ + { + "Name": "Id", + "Type": "int", + "Key": "id" + }, + { + "Name": "Timestamp", + "Type": "DateTime" + }, + { + "Name": "AvgLatency", + "Type": "TimeSpan" + }, + { + "Name": "Venue", + "Type": "string" + }, + { + "Name": "Symbol", + "Type": "string" + }, + { + "Name": "OrderCount", + "Type": "int" + }, + { + "Name": "Volume", + "Type": "long" + } + ] +} diff --git a/src/DataStreamSuite/SharedStreams/RowTemplates/MboSchema.json b/src/DataStreamSuite/SharedStreams/RowTemplates/MboSchema.json new file mode 100644 index 000000000..98c13b23e --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/RowTemplates/MboSchema.json @@ -0,0 +1,54 @@ +{ + "IsTimeSeries": true, + "Columns": [ + { + "Name": "Id", + "Type": "int", + "Key": "id" + }, + { + "Name": "SendTimestamp", + "Type": "DateTime" + }, + { + "Name": "Timestamp", + "Type": "DateTime" + }, + { + "Name": "SequenceNumber", + "Type": "int" + }, + { + "Name": "Venue", + "Type": "string" + }, + { + "Name": "Symbol", + "Type": "string" + }, + { + "Name": "Contract", + "Type": "string" + }, + { + "Name": "Side", + "Type": "bts.api.Side" + }, + { + "Name": "Price", + "Type": "double" + }, + { + "Name": "Quantity", + "Type": "double" + }, + { + "Name": "OrderId", + "Type": "int" + }, + { + "Name": "Action", + "Type": "string" + } + ] +} diff --git a/src/DataStreamSuite/SharedStreams/RowTemplates/MbpSchema.json b/src/DataStreamSuite/SharedStreams/RowTemplates/MbpSchema.json index 94ed92c34..25c3440b8 100644 --- a/src/DataStreamSuite/SharedStreams/RowTemplates/MbpSchema.json +++ b/src/DataStreamSuite/SharedStreams/RowTemplates/MbpSchema.json @@ -1,4 +1,5 @@ { + "IsTimeSeries": true, "Columns": [ { "Name": "Id", @@ -17,85 +18,153 @@ "Name": "Symbol", "Type": "string" }, + { + "Name": "BidPrices", + "Key": null, + "Type": "double[]", + "Value": "{ get; } = new double[5];", + "Equals": "BidPrices.SequenceEqual(other.BidPrices)" + }, + { + "Name": "BidQuantities", + "Key": null, + "Type": "double[]", + "Value": "{ get; } = new double[5];", + "Equals": "BidQuantities.SequenceEqual(other.BidQuantities)" + }, { "Name": "BidPrice1", - "Type": "double" + "Type": "double", + "Value": "BidPrices[0]", + "Equals": null }, { "Name": "BidQuantity1", - "Type": "double" + "Type": "double", + "Value": "BidQuantities[0]", + "Equals": null }, { "Name": "BidPrice2", - "Type": "double" + "Type": "double", + "Value": "BidPrices[1]", + "Equals": null }, { "Name": "BidQuantity2", - "Type": "double" + "Type": "double", + "Value": "BidQuantities[1]", + "Equals": null }, { "Name": "BidPrice3", - "Type": "double" + "Type": "double", + "Value": "BidPrices[2]", + "Equals": null }, { "Name": "BidQuantity3", - "Type": "double" + "Type": "double", + "Value": "BidQuantities[2]", + "Equals": null }, { "Name": "BidPrice4", - "Type": "double" + "Type": "double", + "Value": "BidPrices[3]", + "Equals": null }, { "Name": "BidQuantity4", - "Type": "double" + "Type": "double", + "Value": "BidQuantities[3]", + "Equals": null }, { "Name": "BidPrice5", - "Type": "double" + "Type": "double", + "Value": "BidPrices[4]", + "Equals": null }, { "Name": "BidQuantity5", - "Type": "double" + "Type": "double", + "Value": "BidQuantities[4]", + "Equals": null + }, + { + "Name": "AskPrices", + "Key": null, + "Type": "double[]", + "Value": "{ get; } = new double[5];", + "Equals": "AskPrices.SequenceEqual(other.AskPrices)" + }, + { + "Name": "AskQuantities", + "Key": null, + "Type": "double[]", + "Value": "{ get; } = new double[5];", + "Equals": "AskQuantities.SequenceEqual(other.AskQuantities)" }, { "Name": "AskPrice1", - "Type": "double" + "Type": "double", + "Value": "AskPrices[0]", + "Equals": null }, { "Name": "AskQuantity1", - "Type": "double" + "Type": "double", + "Value": "AskQuantities[0]", + "Equals": null }, { "Name": "AskPrice2", - "Type": "double" + "Type": "double", + "Value": "AskPrices[1]", + "Equals": null }, { "Name": "AskQuantity2", - "Type": "double" + "Type": "double", + "Value": "AskQuantities[1]", + "Equals": null }, { "Name": "AskPrice3", - "Type": "double" + "Type": "double", + "Value": "AskPrices[2]", + "Equals": null }, { "Name": "AskQuantity3", - "Type": "double" + "Type": "double", + "Value": "AskQuantities[2]", + "Equals": null }, { "Name": "AskPrice4", - "Type": "double" + "Type": "double", + "Value": "AskPrices[3]", + "Equals": null }, { "Name": "AskQuantity4", - "Type": "double" + "Type": "double", + "Value": "AskQuantities[3]", + "Equals": null }, { "Name": "AskPrice5", - "Type": "double" + "Type": "double", + "Value": "AskPrices[4]", + "Equals": null }, { "Name": "AskQuantity5", - "Type": "double" + "Type": "double", + "Value": "AskQuantities[4]", + "Equals": null } ] } diff --git a/src/DataStreamSuite/SharedStreams/RowTemplates/OrderImbalanceSchema.json b/src/DataStreamSuite/SharedStreams/RowTemplates/OrderImbalanceSchema.json index 2469f57f1..e4a14c51d 100644 --- a/src/DataStreamSuite/SharedStreams/RowTemplates/OrderImbalanceSchema.json +++ b/src/DataStreamSuite/SharedStreams/RowTemplates/OrderImbalanceSchema.json @@ -1,4 +1,5 @@ { + "IsTimeSeries": true, "Columns": [ { "Name": "Id", @@ -17,115 +18,183 @@ "Name": "Symbol", "Type": "string" }, + { + "Name": "BidPrices", + "Key": null, + "Type": "double[]", + "Value": "{ get; } = new double[5];", + "Equals": "BidPrices.SequenceEqual(other.BidPrices)" + }, + { + "Name": "BidQuantities", + "Key": null, + "Type": "double[]", + "Value": "{ get; } = new double[5];", + "Equals": "BidQuantities.SequenceEqual(other.BidQuantities)" + }, { "Name": "BidPrice1", - "Type": "double" + "Type": "double", + "Value": "BidPrices[0]", + "Equals": null }, { "Name": "BidQuantity1", - "Type": "double" + "Type": "double", + "Value": "BidQuantities[0]", + "Equals": null }, { "Name": "BidPrice2", - "Type": "double" + "Type": "double", + "Value": "BidPrices[1]", + "Equals": null }, { "Name": "BidQuantity2", - "Type": "double" + "Type": "double", + "Value": "BidQuantities[1]", + "Equals": null }, { "Name": "BidPrice3", - "Type": "double" + "Type": "double", + "Value": "BidPrices[2]", + "Equals": null }, { "Name": "BidQuantity3", - "Type": "double" + "Type": "double", + "Value": "BidQuantities[2]", + "Equals": null }, { "Name": "BidPrice4", - "Type": "double" + "Type": "double", + "Value": "BidPrices[3]", + "Equals": null }, { "Name": "BidQuantity4", - "Type": "double" + "Type": "double", + "Value": "BidQuantities[3]", + "Equals": null }, { "Name": "BidPrice5", - "Type": "double" + "Type": "double", + "Value": "BidPrices[4]", + "Equals": null }, { "Name": "BidQuantity5", - "Type": "double" + "Type": "double", + "Value": "BidQuantities[4]", + "Equals": null + }, + { + "Name": "AskPrices", + "Key": null, + "Type": "double[]", + "Value": "{ get; } = new double[5];", + "Equals": "AskPrices.SequenceEqual(other.AskPrices)" + }, + { + "Name": "AskQuantities", + "Key": null, + "Type": "double[]", + "Value": "{ get; } = new double[5];", + "Equals": "AskQuantities.SequenceEqual(other.AskQuantities)" }, { "Name": "AskPrice1", - "Type": "double" + "Type": "double", + "Value": "AskPrices[0]", + "Equals": null }, { "Name": "AskQuantity1", - "Type": "double" + "Type": "double", + "Value": "AskQuantities[0]", + "Equals": null }, { "Name": "AskPrice2", - "Type": "double" + "Type": "double", + "Value": "AskPrices[1]", + "Equals": null }, { "Name": "AskQuantity2", - "Type": "double" + "Type": "double", + "Value": "AskQuantities[1]", + "Equals": null }, { "Name": "AskPrice3", - "Type": "double" + "Type": "double", + "Value": "AskPrices[2]", + "Equals": null }, { "Name": "AskQuantity3", - "Type": "double" + "Type": "double", + "Value": "AskQuantities[2]", + "Equals": null }, { "Name": "AskPrice4", - "Type": "double" + "Type": "double", + "Value": "AskPrices[3]", + "Equals": null }, { "Name": "AskQuantity4", - "Type": "double" + "Type": "double", + "Value": "AskQuantities[3]", + "Equals": null }, { "Name": "AskPrice5", - "Type": "double" + "Type": "double", + "Value": "AskPrices[4]", + "Equals": null }, { "Name": "AskQuantity5", - "Type": "double" + "Type": "double", + "Value": "AskQuantities[4]", + "Equals": null }, { "Name": "Imbalance1", "Type": "double", "Value": "BidQuantity1 - AskQuantity1", - "EqualsOp": null + "Equals": null }, { "Name": "Imbalance2", "Type": "double", "Value": "Imbalance1 + BidQuantity2 - AskQuantity2", - "EqualsOp": null + "Equals": null }, { "Name": "Imbalance3", "Type": "double", "Value": "Imbalance2 + BidQuantity3 - AskQuantity3", - "EqualsOp": null + "Equals": null }, { "Name": "Imbalance4", "Type": "double", "Value": "Imbalance3 + BidQuantity4 - AskQuantity4", - "EqualsOp": null + "Equals": null }, { "Name": "Imbalance5", "Type": "double", "Value": "Imbalance4 + BidQuantity5 - AskQuantity5", - "EqualsOp": null + "Equals": null } ] } diff --git a/src/DataStreamSuite/SharedStreams/RowTemplates/OrderLatencySchema.json b/src/DataStreamSuite/SharedStreams/RowTemplates/OrderLatencySchema.json new file mode 100644 index 000000000..adfb8b02f --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/RowTemplates/OrderLatencySchema.json @@ -0,0 +1,48 @@ +{ + "IsTimeSeries": true, + "Columns": [ + { + "Name": "Id", + "Type": "int", + "Key": "id" + }, + { + "Name": "SendTimestamp", + "Type": "DateTime" + }, + { + "Name": "Timestamp", + "Type": "DateTime" + }, + { + "Name": "Latency", + "Type": "TimeSpan", + "Value": "Timestamp - SendTimestamp", + "Equals": null + }, + { + "Name": "Venue", + "Type": "string" + }, + { + "Name": "Symbol", + "Type": "string" + }, + { + "Name": "Price", + "Type": "double" + }, + { + "Name": "Quantity", + "Type": "double" + }, + { + "Name": "Flags", + "Type": "string" + }, + { + "Name": "SequenceNumber", + "Type": "int" + } + ] +} diff --git a/src/DataStreamSuite/SharedStreams/RowTemplates/OrderLifespanSchema.json b/src/DataStreamSuite/SharedStreams/RowTemplates/OrderLifespanSchema.json new file mode 100644 index 000000000..01614f3f6 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/RowTemplates/OrderLifespanSchema.json @@ -0,0 +1,62 @@ +{ + "IsTimeSeries": true, + "Columns": [ + { + "Name": "Id", + "Type": "int", + "Key": "id" + }, + { + "Name": "Venue", + "Type": "string" + }, + { + "Name": "Symbol", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "DateTime" + }, + { + "Name": "SendTimestamp", + "Type": "DateTime" + }, + { + "Name": "Latency", + "Type": "TimeSpan", + "Value": "Timestamp - SendTimestamp", + "Equals": null + }, + { + "Name": "FinalTimestamp", + "Type": "DateTime" + }, + { + "Name": "Lifespan", + "Type": "TimeSpan", + "Value": "FinalTimestamp - Timestamp", + "Equals": null + }, + { + "Name": "State", + "Type": "string" + }, + { + "Name": "Side", + "Type": "bts.api.Side" + }, + { + "Name": "Price", + "Type": "double" + }, + { + "Name": "Quantity", + "Type": "double" + }, + { + "Name": "FilledQuantity", + "Type": "double" + } + ] +} diff --git a/src/DataStreamSuite/SharedStreams/RowTemplates/RefTypeRow.tt b/src/DataStreamSuite/SharedStreams/RowTemplates/RefTypeRow.tt index dde5841bb..70aef9810 100644 --- a/src/DataStreamSuite/SharedStreams/RowTemplates/RefTypeRow.tt +++ b/src/DataStreamSuite/SharedStreams/RowTemplates/RefTypeRow.tt @@ -31,6 +31,7 @@ } bool isTimeSeries = schemaJ.TryGetPropertyValue("IsTimeSeries", out jn) && jn.GetValue(); #> +#nullable disable namespace bts.SilverPine.Tables; using BCG; diff --git a/src/DataStreamSuite/SharedStreams/RowTemplates/RefTypeVariableColumnRow.tt b/src/DataStreamSuite/SharedStreams/RowTemplates/RefTypeVariableColumnRow.tt new file mode 100644 index 000000000..b58131bf5 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/RowTemplates/RefTypeVariableColumnRow.tt @@ -0,0 +1,152 @@ +<#@ template language="C#" #> +<#@ output extension=".cs" #> +<#@ parameter name="TableName" type="System.String" #> +<#@ parameter name="TypeName" type="System.String" #> +<#@ parameter name="SchemaJson" type="System.String" #> +<#@ import namespace="System" #> +<#@ import namespace="System.Text.Json" #> +<#@ import namespace="System.Text.Json.Nodes" #> +<# + System.Console.WriteLine($"Reading json file '{SchemaJson}'"); + var schemaStr = System.IO.File.ReadAllText(SchemaJson); + System.Console.WriteLine("Dingo"); + var schemaJ = JsonSerializer.Deserialize(schemaStr); + if (schemaJ is null) { + throw new ArgumentException($"Failed to parse json from '{schemaJ}'"); + } + System.Console.WriteLine("Blingo"); + if (!schemaJ.TryGetPropertyValue("Columns", out JsonNode? jn)) { + throw new ArgumentException($"Failed to get 'Columns' field from '{schemaJ}'"); + } + var columnsJArray = jn.AsArray(); + if (columnsJArray is null) { + throw new ArgumentException($"Failed to get 'Columns' field from '{schemaJ}'"); + } + System.Console.WriteLine("Ringo"); + var columns = new System.Collections.Generic.List>(); + for (int colIdx = 0; colIdx < columnsJArray.Count; colIdx++) { + var columnJ = columnsJArray[colIdx].AsObject(); + string name = columnJ["Name"].GetValue(); + System.Console.WriteLine($"Adding prop {name}"); + string type = columnJ["Type"]?.GetValue(); + string key = columnJ.TryGetPropertyValue("Key", out var keyJ) ? keyJ?.GetValue() : name; + string value = null; + if (columnJ.TryGetPropertyValue("Value", out var valueJ)) { + value = valueJ.GetValue().Trim(); + if (!value.StartsWith('{')) { + value = $"=> {valueJ.GetValue()};"; + } + } else { + value = "{ get; init; }"; + } + string equalsOp = null; + if (columnJ.TryGetPropertyValue("Equals", out var equalsJ)) { + equalsOp = equalsJ?.GetValue(); + } else { + equalsOp = $"EqualityComparer<{type}>.Default.Equals({name}, other.{name})"; + } + columns.Add(new(name, type, key, value, equalsOp)); + } + bool isTimeSeries = schemaJ.TryGetPropertyValue("IsTimeSeries", out jn) && jn.GetValue(); +#> +#nullable disable +namespace bts.SilverPine.Tables; + +using BCG; + +using System; + +public class <#=TypeName#> : <#=isTimeSeries ? "ITimeSeriesRow" : "IBlueRow" #>, IEquatable<<#=TypeName#>> { + public static IBlueTableSchema Schema { get; } = + new BlueTableSchema( + new ColumnInfo[] { + <# + // Generate properties + int offset = 0; + for (int i = 0; i < columns.Count; i++) { + var type = columns[i].Item2; + var key = columns[i].Item3; + if (key is null) { + offset++; + continue; + } + #> + new("<#=key#>", <#=i - offset#>, typeof(<#=type#>)), + <# + } + #> + }, + "<#= TableName #>"); + IBlueTableSchema IBlueRow.schema => Schema; + + <# + // Generate properties + foreach (var column in columns) { + var name = column.Item1; + var type = column.Item2; + var getter = column.Item4; + #> +public <#= type #> <#= name #> <#= getter #> + <# + } + #> + + // Indexer by column name + public object? this[string col] => col switch { + <# + // Generate case statements for each column name + for (int i = 0; i < columns.Count; i++) { + var name = columns[i].Item1; + var key = columns[i].Item3; + if (key is null) { + continue; + } + #> +"<#= key #>" => <#= name #>, + <# +} + #> +_ => throw new IndexOutOfRangeException() + }; + + // Indexer by column index + public object? this[int col] => col switch { + <# + // Generate case statements for each column index + offset = 0; + for (int i = 0; i < columns.Count; i++) { + var name = columns[i].Item1; + var key = columns[i].Item3; + if (key is null) { + offset++; + continue; + } + #> +<#= i - offset #> => <#= name #>, + <# +} + #> +_ => throw new KeyNotFoundException() + }; + + public bool Equals(<#= TypeName #> other) => + other is not null + <# + // Generate properties + for (int i = 0; i < columns.Count; i++) { + var equalsOp = columns[i].Item5; + if (string.IsNullOrEmpty(equalsOp)) { + continue; + } + equalsOp = equalsOp.Insert(0, "&& "); + #> + <#=equalsOp#> + <#}#> + ; + + public override bool Equals(object? obj) => + obj is <#=TypeName#> other && Equals(other); + + public override int GetHashCode() => + Id; +} diff --git a/src/DataStreamSuite/SharedStreams/SharedStreams.csproj b/src/DataStreamSuite/SharedStreams/SharedStreams.csproj index c6109c103..ab43fa2cb 100644 --- a/src/DataStreamSuite/SharedStreams/SharedStreams.csproj +++ b/src/DataStreamSuite/SharedStreams/SharedStreams.csproj @@ -23,8 +23,12 @@ - - + + + + + + diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/AvgOrderLatencyStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/AvgOrderLatencyStreamFactory.cs new file mode 100644 index 000000000..1b9964b87 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/AvgOrderLatencyStreamFactory.cs @@ -0,0 +1,106 @@ +namespace bts.SilverPine; + +using BCG; + +using bts.api.core; +using bts.SilverPine.StreamFactory.Logging; +using bts.SilverPine.Tables; + +using Bts.DataStream; + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using System; +using System.Reactive.Linq; + +public class AvgOrderLatencyStreamFactory : IDataStreamFactory { + readonly DatabaseConfig config; + readonly ILogger logger; + + public AvgOrderLatencyStreamFactory(IOptions config, ILogger logger) + : this(config.Value, logger) { } + + public AvgOrderLatencyStreamFactory(DatabaseConfig config, ILogger logger) { + this.config = config; + this.logger = logger; + } + + public static IObservable>? CreateAvgOrderLatencyTables( + DatabaseConfig config, HistoricDataParams args, ILogger logger) { + args.Validate().ValidateInterval(); + var orderLatencyParams = new OrderLatencyStreamParams() { + Venue = args.Venue, + Symbol = args.Symbol, + BeginDate = args.BeginDate, + EndDate = args.EndDate, + }; + return OrderLatencyStreamFactory.CreateOrderLatencyTables(config, orderLatencyParams, logger) + ?.BufferTable(TimeSpan.FromMilliseconds(500)) + ?.GroupTable( + r => r.Id, + r => (int)((r.Timestamp.Ticks - args.BeginDate.Ticks) / args.Interval!.Value.Ticks)) + ?.SelectKeyValue(ComputeAvgLatency, r => r.Id); + + AvgOrderLatencyRow ComputeAvgLatency(TableUpdateGrouping group) { + double avgLatency = 0; + double latencySum = 0; + int nTotal = 0; + int n = 0; + foreach (var r in group.table.state.Values) { + if ((double.MaxValue - latencySum) > r.Latency.Ticks) { + latencySum += r.Latency.Ticks; + n++; + } else if (n > 0) { + nTotal += n; + avgLatency += ((latencySum / n) - avgLatency) * (n / nTotal); + latencySum = 0; + n = 0; + } else { + throw new InvalidOperationException("Latency is too long (unable to compute the avg)"); + } + } + if (n > 0) { + nTotal += n; + avgLatency += ((latencySum / n) - avgLatency) * (n / nTotal); + latencySum = 0; + n = 0; + } + return new AvgOrderLatencyRow() { + Symbol = args.Symbol, + Venue = args.Venue, + Id = group.group_key, + AvgLatency = new((long)avgLatency), + Timestamp = args.BeginDate + new TimeSpan(group.group_key * args.Interval!.Value.Ticks), + OrderCount = group.table.state.Count, + Volume = (long)group.table.state.Values.Sum(r => r.Quantity) + }; + } + } + + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (SupportedStreams.AvgOrderLatency.IsMatch(streamId)) { + return createOrderLatency(args); + } + return null; + } + + public IEnumerable GetSupportedStreams() { + yield return SupportedStreams.AvgOrderLatency; + } + + ViewStream? createOrderLatency(IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (!StreamFactoryArgStore.TryGetParams(args, out OrderLatencyStreamParams? sp, out var err)) { + logger.LogInvalidParams("avgOrderLatency", err); + return null; + } + return OrderLatencyStreamFactory.CreateOrderLatencyTables(config, sp, logger) + ?.AsViewStream(AvgOrderLatencyRow.Schema); + } + + static class SupportedStreams { + public static readonly DataStreamMetadata AvgOrderLatency = new() { + Name = AvgOrderLatencyRow.Schema.tableName + }; + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/AvgOrderLatencyStreamParams.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/AvgOrderLatencyStreamParams.cs new file mode 100644 index 000000000..f6b4c576e --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/AvgOrderLatencyStreamParams.cs @@ -0,0 +1,34 @@ +namespace bts.SilverPine; + +using System.Diagnostics.CodeAnalysis; + +public class AvgOrderLatencyStreamParams { + [StreamArgument("venue", "exchange", "exch", IsRequired = true)] + public string? Venue { get; set; } + + [StreamArgument("symbol", "sym", "product", "asset", IsRequired = true)] + public string? Symbol { get; set; } + + [StreamArgument("begin", "beginDate", "start", "startDate", "from", IsRequired = true)] + public DateTime BeginDate { get; set; } + + [StreamArgument("end", "endDate", "stop", "stopDate", "to", IsRequired = true)] + public DateTime EndDate { get; set; } + + [StreamArgument("interval", IsRequired = false)] + public TimeSpan Interval { get; set; } = TimeSpan.Zero; + + [MemberNotNull(nameof(Venue), nameof(Symbol))] + public AvgOrderLatencyStreamParams Validate() { + if (Venue is null) { + throw new InvalidOperationException("Venue is null"); + } + if (Symbol is null) { + throw new InvalidOperationException("Symbol is null"); + } + if (Interval < TimeSpan.Zero) { + throw new InvalidOperationException("Interval < TimeSpan.Zero"); + } + return this; + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/BboStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/BboStreamFactory.cs index 065f07460..f85a58678 100644 --- a/src/DataStreamSuite/SharedStreams/StreamFactories/BboStreamFactory.cs +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/BboStreamFactory.cs @@ -13,7 +13,6 @@ using Microsoft.Extensions.Options; using System; using System.Collections.Generic; -using System.Data; public class BboStreamFactory : IDataStreamFactory { readonly DatabaseConfig config; @@ -27,23 +26,19 @@ public class BboStreamFactory : IDataStreamFactory { this.logger = logger; } - public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { - if (SupportedStreams.Bbo.IsMatch(streamId)) { - return createBbo(args); - } - return null; - } - - public IObservable>? CreateTables( - BboStreamParams args) { + public static IObservable>? CreateBboTables(DatabaseConfig config, BboStreamParams args, ILogger logger) { args.Validate(); return new HistoricDataLoader(config, logger).LoadTablesObservable( - args.DataType, args.Venue, args.Symbol, args.BeginDate, args.EndDate, args.Interval, GetTimestamp, RowFactory); - - DateTime? GetTimestamp(IBlueRow r) { - var tsObj = r["ts_event"]; - return tsObj is not DateTime ts && !DateTime.TryParse(tsObj as string, out ts) ? null : ts; - } + new HistoricDataParams() { + DataType = args.DataType, + Venue = args.Venue, + Symbol = args.Symbol, + BeginDate = args.BeginDate, + EndDate = args.EndDate, + Interval = args.Interval, + TsNanosCol = "ts_event_ns" + }, + RowFactory); BboRow RowFactory(IBlueRow row, int groupKey, DateTime timestamp) => new() { @@ -51,13 +46,20 @@ public class BboStreamFactory : IDataStreamFactory { Timestamp = timestamp, Symbol = args.Symbol, Venue = args.Venue, - BidPrice = row.GetDouble("bid_px_00") ?? double.NaN, - BidQuantity = row.GetDouble("bid_sz_00") ?? double.NaN, - AskPrice = row.GetDouble("ask_px_00") ?? double.NaN, - AskQuantity = row.GetDouble("ask_sz_00") ?? double.NaN, + BidPrice = row.GetFloat64("bid_px_00") ?? double.NaN, + BidQuantity = row.GetFloat64("bid_sz_00") ?? double.NaN, + AskPrice = row.GetFloat64("ask_px_00") ?? double.NaN, + AskQuantity = row.GetFloat64("ask_sz_00") ?? double.NaN, }; } + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (SupportedStreams.Bbo.IsMatch(streamId)) { + return createBbo(args); + } + return null; + } + public IEnumerable GetSupportedStreams() { yield return SupportedStreams.Bbo; } @@ -67,7 +69,7 @@ public class BboStreamFactory : IDataStreamFactory { logger.LogInvalidParams("bbo", err); return null; } - return CreateTables(bsp)?.AsViewStream(BboRow.Schema); + return CreateBboTables(config, bsp, logger)?.AsViewStream(BboRow.Schema); } static class SupportedStreams { diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/HistoricDataLoader.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/HistoricDataLoader.cs index 02f69138c..0e7cee2b4 100644 --- a/src/DataStreamSuite/SharedStreams/StreamFactories/HistoricDataLoader.cs +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/HistoricDataLoader.cs @@ -5,19 +5,21 @@ using BCG; using bts.api.core; using Bts.DataStream; +using Bts.DataStreamCore; using Microsoft.Extensions.Logging; using System.Data; using System.Data.SQLite; +using System.Linq; using System.Reactive.Linq; using System.Reflection; public class HistoricDataLoader { readonly DatabaseConfig dbConfig; - readonly ILogger logger; + readonly ILogger? logger; - public HistoricDataLoader(DatabaseConfig dbConfig, ILogger logger) { + public HistoricDataLoader(DatabaseConfig dbConfig, ILogger? logger) { this.dbConfig = dbConfig; this.logger = logger; } @@ -42,56 +44,77 @@ public class HistoricDataLoader { Query = r["query"] as string, }); } catch (Exception ex) { - logger.LogError("Failed to load query metadata from '{dbPath}'. Query: {query}. Exception {ex}", dbConfig.Path, dbConfig.Query, ex); + logger?.LogError("Failed to load query metadata from '{dbPath}'. Query: {query}. Exception {ex}", dbConfig.Path, dbConfig.Query, ex); } return Enumerable.Empty(); } - public IAsyncEnumerable LoadRowsAsync(string dataType, string venue, string? symbol, DateTime beginDate, DateTime endDate) { - venue = resolveVenue(venue); + public IAsyncEnumerable LoadRowsAsync(HistoricDataParams args) { + args.Validate(); + var venue = resolveVenue(args.Venue); var queryMetadatas = LoadQueryMetadata().ToList(); var perDateQueries = queryMetadatas - .Where(m => m.DataType == dataType && m.Symbol == symbol && m.BeginDate < endDate && m.EndDate > beginDate) + .Where(m => m.DataType == args.DataType && m.Symbol == args.Symbol && m.BeginDate < args.EndDate && m.EndDate > args.BeginDate) .ToList(); if (perDateQueries.Count == 0) { return AsyncEnumerable.Empty(); } return perDateQueries.ToAsyncEnumerable().SelectMany(q => { - var location = q.Location ?? throw new InvalidOperationException($"'location' is null in metadata for {dataType} stream"); - var query = q.Query ?? throw new InvalidOperationException($"'query' is null in metadata for {dataType} stream"); + var location = q.Location ?? throw new InvalidOperationException($"'location' is null in metadata for {args.DataType} stream"); if (!Path.IsPathRooted(location)) { var dir = Path.GetDirectoryName(dbConfig.Path) ?? Assembly.GetExecutingAssembly().Location; location = Path.Combine(dir, location); } + var query = q.Query ?? throw new InvalidOperationException($"'query' is null in metadata for {args.DataType} stream"); + var andFilters = new List(2); + if (args.BeginDate > DateTime.MinValue) { + var unixTicks = args.BeginDate.Ticks - DateTime.UnixEpoch.Ticks; + andFilters.Add($"{args.TsNanosCol} > {unixTicks * 100 - 1}"); + } + if (args.EndDate < DateTime.MaxValue) { + var unixTicks = args.EndDate.Ticks - DateTime.UnixEpoch.Ticks; + andFilters.Add($"{args.TsNanosCol} < {unixTicks * 100}"); + } + if (andFilters.Count > 0) { + if (query.EndsWith(';')) { + query = query.Substring(0, query.Length - 1); + } + query += " where " + string.Join(" and ", andFilters) + ';'; + } return new SQLiteEnumerable(location, query); }); } - public IObservable>? LoadTablesObservable( - string dataType, - string venue, - string symbol, - DateTime beginDate, - DateTime endDate, - TimeSpan interval, - Func getTimestamp, - Func rowFactory - ) where T : ITimeSeriesRow { - return LoadRowsAsync(dataType, venue, symbol, beginDate, endDate) - .ToObservable() - .Scan(TableUpdate.mk_empty(r => r.Id), AddPage); + public IObservable>? LoadTablesObservable(HistoricDataParams args, Func rowFactory) where T : ITimeSeriesRow { + args.Validate(); + var seed = TableUpdate.mk_empty( + args.Interval.HasValue + ? r => GroupKey(r.Timestamp) + : r => r.Id); + return LoadRowsAsync(args).ToObservable().Scan(seed, AddPage); + + int GroupKey(DateTime ts) => + (int)((ts.Ticks - args.BeginDate.Ticks) / args.Interval!.Value.Ticks); TableUpdate AddPage(TableUpdate res, IBlueRowCollection blueRows) { + var tsNanosIdx = blueRows.schema.ResolveIndices(args.TsNanosCol); var bd = res.to_builder(); foreach (IBlueRow blueRow in blueRows) { - var ts = getTimestamp(blueRow); - if (!ts.HasValue || ts < beginDate || ts >= endDate) { + var tsNanos = blueRow.GetInt64(tsNanosIdx); + if (!tsNanos.HasValue) { continue; } - var delta = ts - beginDate; - var groupKey = (int)(delta / interval); + var ts = DateTime.UnixEpoch.Add(new TimeSpan(tsNanos.Value / 100)); + if (ts < args.BeginDate || ts >= args.EndDate) { + continue; + } + if (!args.Interval.HasValue) { + bd.add_or_update(rowFactory(blueRow, blueRow.Id(), ts)); + continue; + } + var groupKey = GroupKey(ts); if (!bd.TryGetValue(groupKey, out var row) || row.Timestamp < ts) { - row = rowFactory(blueRow, groupKey, ts.Value); + row = rowFactory(blueRow, groupKey, ts); if (row is null) { continue; } diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/HistoricDataParams.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/HistoricDataParams.cs new file mode 100644 index 000000000..cd8cc998d --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/HistoricDataParams.cs @@ -0,0 +1,60 @@ +namespace bts.SilverPine; + +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +public interface IHistoricDataParams { + string? DataType { get; } + string? Venue { get; } + string? Symbol { get; } + DateTime BeginDate { get; } + DateTime EndDate { get; } + TimeSpan? Interval { get; } + string? TsNanosCol { get; } +} + +public class HistoricDataParams : IHistoricDataParams { + public string? DataType { get; set; } + public string? Venue { get; set; } + public string? Symbol { get; set; } + public DateTime BeginDate { get; set; } + public DateTime EndDate { get; set; } + public TimeSpan? Interval { get; set; } + public string? TsNanosCol { get; set; } + + [MemberNotNull(nameof(DataType), nameof(Venue), nameof(Symbol), nameof(TsNanosCol))] + public HistoricDataParams Validate() { + validateNotNull(DataType); + validateNotNull(Venue); + validateNotNull(Symbol); + validateNotNull(TsNanosCol); + return this; + } + + [MemberNotNull(nameof(Interval))] + public HistoricDataParams ValidateInterval() { + validateNotNull(Interval); + return this; + } + + static void validateNotNull([NotNull]T arg, [CallerArgumentExpression(nameof(arg))]string argName = "") { + if (arg is null) { + throw new InvalidOperationException($"Invalid HistoricDataParams: {argName} is null"); + } + } +} + +public static class HistoricDataParamsExtensions { + // public static void Validate(this IHistoricDataParams hdp) { + // validateNotNull(hdp.DataType); + // validateNotNull(hdp.Venue); + // validateNotNull(hdp.Symbol); + // validateNotNull(hdp.TsNanosCol); + // } + + // static void validateNotNull([NotNull]T arg, [CallerArgumentExpression(nameof(arg))]string argName = "") { + // if (arg is null) { + // throw new InvalidOperationException($"Invalid HistoricDataParams: {argName} is null"); + // } + // } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/MboStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/MboStreamFactory.cs new file mode 100644 index 000000000..7b4f9bd39 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/MboStreamFactory.cs @@ -0,0 +1,121 @@ +namespace bts.SilverPine; + +using BCG; + +using bts.api; +using bts.api.core; +using bts.SilverPine.StreamFactory.Logging; +using bts.SilverPine.Tables; + +using Bts.DataStream; + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using System.Collections.Generic; +using System.Reactive.Linq; + +public class MboStreamFactory : IDataStreamFactory { + readonly DatabaseConfig config; + ILogger logger; + + public MboStreamFactory(IOptions config, ILogger logger) + : this(config.Value, logger) { } + + public MboStreamFactory(DatabaseConfig config, ILogger logger) { + this.config = config; + this.logger = logger; + } + + public static IObservable>? CreateTables( + DatabaseConfig config, string venue, string symbol, DateTime beginDate, DateTime endDate) { + return new HistoricDataLoader(config, null) + .LoadRowsAsync(new HistoricDataParams { + DataType = "mbo", + Venue = venue, + Symbol = symbol, + BeginDate = beginDate, + EndDate = endDate, + TsNanosCol = "ts_recv_ns"}) + .ToObservable() + .Scan(TableUpdate.mk_empty(r => r.Id), AddPage) + .Where(t => t.has_changes()); + + TableUpdate AddPage(TableUpdate res, IBlueRowCollection rows) { + var bd = res.to_builder(); + int i = res.state.Count; + foreach (IBlueRow br in rows) { + var sendTimestampNanos = br.get("ts_event_ns"); + var recvTimestampNanos = br.get("ts_recv_ns"); + var sideStr = br.get("side"); + var side = sideStr[0] switch { + 'A' => Side.Sell, + 'B' => Side.Buy, + 'N' => Side.Undefined, + _ => throw new InvalidOperationException($"Unsupported side '{sideStr}'"), + }; + var r = new MboRow() { + Id = i++, + SendTimestamp = DateTime.UnixEpoch.Add(new TimeSpan(sendTimestampNanos / 100)), + Timestamp = DateTime.UnixEpoch.Add(new TimeSpan(recvTimestampNanos / 100)), + SequenceNumber = br.get("sequence"), + Venue = venue, + Symbol = symbol, + Contract = symbol, + Side = side, + Price = br.get("price", double.NaN), + Quantity = br.get("size"), + OrderId = br.get("order_id"), + Action = br.get("action") + }; + bd.add_or_update(r); + } + return bd.to_table(); + } + } + + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (SupportedStreams.Mbo.IsMatch(streamId)) { + return createMbo(args); + } + return null; + } + + public IEnumerable GetSupportedStreams() { + yield return SupportedStreams.Mbo; + } + + ViewStream? createMbo(IEnumerable<(string? Name, int? Index, object? Value)> args) { + var argStore = new StreamFactoryArgStore(args); + var venue = argStore.GetVenueString(); + if (venue is null) { + logger.LogMissingParam("mbo", "venue"); + return null; + } + var symbol = argStore.GetSymbol(); + if (symbol is null) { + logger.LogMissingParam("mbo", "symbol"); + return null; + } + var begin = argStore.GetDateTime("begin"); + if (!begin.HasValue) { + logger.LogMissingParam("mbo", "begin"); + return null; + } + var end = argStore.GetDateTime("end"); + if (!end.HasValue) { + logger.LogMissingParam("mbo", "end"); + return null; + } + var vs = CreateTables(config, venue, symbol, begin.Value, end.Value) + ?.AsDataPackets(MboRow.Schema) + ?.AsViewStream(); + return vs; + } + + static class SupportedStreams { + public static readonly DataStreamMetadata Mbo = new() { + Name = "mbo" + }; + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/MboStreamParams.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/MboStreamParams.cs new file mode 100644 index 000000000..cfa22b188 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/MboStreamParams.cs @@ -0,0 +1,28 @@ +namespace bts.SilverPine; + +using System.Diagnostics.CodeAnalysis; + +public class MboStreamParams { + [StreamArgument("venue", "exchange", "exch", IsRequired = true)] + public string? Venue { get; set; } + + [StreamArgument("symbol", "sym", "product", "asset", IsRequired = true)] + public string? Symbol { get; set; } + + [StreamArgument("begin", "beginDate", "start", "startDate", "from", IsRequired = true)] + public DateTime BeginDate { get; set; } + + [StreamArgument("end", "endDate", "stop", "stopDate", "to", IsRequired = true)] + public DateTime EndDate { get; set; } + + [MemberNotNull(nameof(Venue), nameof(Symbol))] + public MboStreamParams Validate() { + if (Venue is null) { + throw new InvalidOperationException("Venue is null"); + } + if (Symbol is null) { + throw new InvalidOperationException("Symbol is null"); + } + return this; + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/MbpStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/MbpStreamFactory.cs index c606c73de..5663f1204 100644 --- a/src/DataStreamSuite/SharedStreams/StreamFactories/MbpStreamFactory.cs +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/MbpStreamFactory.cs @@ -6,7 +6,6 @@ using bts.api.core; using bts.SilverPine.StreamFactory.Logging; using bts.SilverPine.Tables; - using Bts.DataStream; using Bts.DataStreamCore; @@ -16,7 +15,6 @@ using Microsoft.Extensions.Options; using Spark.Core; using System.Reactive.Linq; -using System.Reflection; public class MbpStreamFactory : IDataStreamFactory { readonly DatabaseConfig config; @@ -30,24 +28,23 @@ public class MbpStreamFactory : IDataStreamFactory { this.logger = logger; } - public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { - if (SupportedStreams.Mbp.IsMatch(streamId)) { - return create(args); - } - if (SupportedStreams.OrderImbalance.IsMatch(streamId)) { - return create(args); - } - return null; - } - - public static IObservable>? CreateTables( - DatabaseConfig config, string venue, string symbol, DateTime beginDate, DateTime endDate, TimeSpan interval, int numLevels) { - return loadPagesAsync(config, venue, symbol, beginDate, endDate) + public static IObservable>? CreateTables( + DatabaseConfig config, MbpStreamParams args, ILogger logger) { + args.Validate(); + return new HistoricDataLoader(config, logger) + .LoadRowsAsync(new HistoricDataParams() { + DataType = "mbp10", + Venue = args.Venue, + Symbol = args.Symbol, + BeginDate = args.BeginDate, + EndDate = args.EndDate, + TsNanosCol = "ts_event_ns" + }) .ToObservable() - .Scan(TableUpdate.mk_empty(r => r.Id), AddPage) + .Scan(TableUpdate.mk_empty(r => r.Id), AddPage) .Where(t => t.has_changes()); - TableUpdate AddPage(TableUpdate res, IBlueRowCollection rows) { + TableUpdate AddPage(TableUpdate res, IBlueRowCollection rows) { var bd = res.to_builder(); var refDate = new DateTime(2000, 1, 1); foreach (IBlueRow r in rows) { @@ -58,19 +55,19 @@ public class MbpStreamFactory : IDataStreamFactory { if (ts < refDate) { continue; } - var groupKey = (int)((ts - refDate) / interval); + var groupKey = (int)((ts - refDate) / args.Interval); if (!res.state.TryGetValue(groupKey, out var bboRow) || bboRow.Timestamp > ts) { - var mbpRow = new MbpVariableDepthRow(numLevels) { + var mbpRow = new MbpRow() { Id = r.Id(), Timestamp = r.get_datetime("ts_event"), - Venue = venue, - Symbol = symbol, + Venue = args.Venue, + Symbol = args.Symbol, }; - for (int i = 0; i < numLevels; i++) { - mbpRow.BidQuantities[i] = r.GetDouble($"bid_sz_{i:00}") ?? 0; - mbpRow.BidPrices[i] = mbpRow.BidQuantities[i] > 0 ? r.GetDouble($"bid_px_{i:00}").assert_value() : double.NaN; - mbpRow.AskQuantities[i] = r.GetDouble($"ask_sz_{i:00}") ?? 0; - mbpRow.AskPrices[i] = mbpRow.AskQuantities[i] > 0 ? r.GetDouble($"ask_px_{i:00}").assert_value() : double.NaN; + for (int i = 0; i < args.NumLevels; i++) { + mbpRow.BidQuantities[i] = r.GetFloat64($"bid_sz_{i:00}") ?? 0; + mbpRow.BidPrices[i] = mbpRow.BidQuantities[i] > 0 ? r.GetFloat64($"bid_px_{i:00}").assert_value() : double.NaN; + mbpRow.AskQuantities[i] = r.GetFloat64($"ask_sz_{i:00}") ?? 0; + mbpRow.AskPrices[i] = mbpRow.AskQuantities[i] > 0 ? r.GetFloat64($"ask_px_{i:00}").assert_value() : double.NaN; } bd.add_or_update(mbpRow); } @@ -79,107 +76,28 @@ public class MbpStreamFactory : IDataStreamFactory { } } - static string resolveVenue(string venue) => - venue switch { - "CME" or "XCME" or "CBOT" or "XCBT" => "GLBX", - _ => venue - }; - - static IEnumerable loadQueryMetadata(DatabaseConfig config, ILogger? logger = null) { - var dbPath = config.Path; - var query = config.Query; - if (dbPath is not null && query is not null) { - try { - return StreamQueryMetadata.Load(dbPath, query); - } catch (Exception ex) { - logger?.LogError("Failed to load query metadata from '{dbPath}'. Exception {ex}", dbPath, ex); - } + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (SupportedStreams.Mbp.IsMatch(streamId)) { + return createMbp(args); } - return Enumerable.Empty(); - } - - ViewStream? create(IEnumerable<(string? Name, int? Index, object? Value)> args) { - var argStore = new StreamFactoryArgStore(args); - var venue = argStore.GetVenueString(); - if (venue is null) { - logger.LogMissingParam("bbo", "venue"); - return null; - } - var symbol = argStore.GetSymbol(); - if (symbol is null) { - logger.LogMissingParam("bbo", "symbol"); - return null; - } - var begin = argStore.GetDateTime("begin"); - if (!begin.HasValue) { - logger.LogMissingParam("bbo", "begin"); - return null; - } - var end = argStore.GetDateTime("end"); - if (!end.HasValue) { - logger.LogMissingParam("bbo", "end"); - return null; - } - var interval = argStore.GetTimeSpan("interval"); - if (!interval.HasValue) { - logger.LogMissingParam("bbo", "interval"); - return null; - } - var numLevels = argStore.GetInt32("levels") ?? 1; - var schema = MbpVariableDepthRow.GetSchema(numLevels); - var vs = CreateTables(config, venue, symbol, begin.Value, end.Value, interval.Value, numLevels) - ?.AsDataPackets(schema) - ?.AsViewStream(); - return vs; + return null; } public IEnumerable GetSupportedStreams() { yield return SupportedStreams.Mbp; } - static IAsyncEnumerable loadPagesAsync(DatabaseConfig config, string venue, string? symbol, DateTime beginDate, DateTime endDate) { - const string dataType = "mbp10"; - venue = resolveVenue(venue); - var queryMetadatas = loadQueryMetadata(config).ToList(); - var perDateQueries = queryMetadatas - .Where(m => m.DataType == dataType && m.Symbol == symbol && m.BeginDate < endDate && m.EndDate > beginDate) - .ToList(); - if (perDateQueries.Count == 0) { - return AsyncEnumerable.Empty(); + ViewStream? createMbp(IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (!StreamFactoryArgStore.TryGetParams(args, out MbpStreamParams? msp, out var err)) { + logger.LogInvalidParams("mbp", err); + return null; } - return perDateQueries.ToAsyncEnumerable().SelectMany(q => { - var location = q.Location ?? throw new InvalidOperationException($"'location' is null in metadata for {dataType} stream"); - if (!Path.IsPathRooted(location)) { - var dir = Path.GetDirectoryName(config.Path) ?? Assembly.GetExecutingAssembly().Location; - location = Path.Combine(dir, location); - } - var query = q.Query ?? throw new InvalidOperationException($"'query' is null in metadata for {dataType} stream"); - var andFilters = new List(2); - if (beginDate > DateTime.MinValue) { - var unixTicks = beginDate.Ticks - DateTime.UnixEpoch.Ticks; - andFilters.Add($"ts_event_ns > {unixTicks * 100 - 1}"); - } - if (endDate < DateTime.MaxValue) { - var unixTicks = endDate.Ticks - DateTime.UnixEpoch.Ticks; - andFilters.Add($"ts_event_ns < {unixTicks * 100}"); - } - if (andFilters.Count > 0) { - if (query.EndsWith(';')) { - query = query.Substring(0, query.Length - 1); - } - query += " where " + string.Join(" and ", andFilters) + ';'; - } - return new SQLiteEnumerable(location, query); - }); + return CreateTables(config, msp, logger)?.AsViewStream(MbpRow.Schema); } - static class SupportedStreams { public static readonly DataStreamMetadata Mbp = new() { Name = "mbp" }; - public static readonly DataStreamMetadata OrderImbalance = new() { - Name = OrderImbalanceRow.Schema.tableName - }; } } diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/MbpStreamParams.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/MbpStreamParams.cs new file mode 100644 index 000000000..bb7dfd30b --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/MbpStreamParams.cs @@ -0,0 +1,37 @@ +namespace bts.SilverPine; + +using System.Diagnostics.CodeAnalysis; + +public class MbpStreamParams { + [StreamArgument("venue", "exchange", "exch", IsRequired = true)] + public string? Venue { get; set; } + + [StreamArgument("symbol", "sym", "product", "asset", IsRequired = true)] + public string? Symbol { get; set; } + + [StreamArgument("begin", "beginDate", "start", "startDate", "from", IsRequired = true)] + public DateTime BeginDate { get; set; } + + [StreamArgument("end", "endDate", "stop", "stopDate", "to", IsRequired = true)] + public DateTime EndDate { get; set; } + + [StreamArgument("interval", IsRequired = true)] + public TimeSpan Interval { get; set; } + + [StreamArgument("levels")] + public int NumLevels { get; set; } + + [MemberNotNull(nameof(Venue), nameof(Symbol))] + public MbpStreamParams Validate() { + if (Venue is null) { + throw new InvalidOperationException("Venue is null"); + } + if (Symbol is null) { + throw new InvalidOperationException("Symbol is null"); + } + if (Interval <= TimeSpan.Zero) { + throw new InvalidOperationException("Interval <= TimeSpan.Zero"); + } + return this; + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/MbpVariableDepthRow.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/MbpVariableDepthRow.cs index b9e499184..21fc7f167 100644 --- a/src/DataStreamSuite/SharedStreams/StreamFactories/MbpVariableDepthRow.cs +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/MbpVariableDepthRow.cs @@ -4,7 +4,7 @@ using BCG; using Bts.DataStreamCore; -public struct MbpVariableDepthRow : IBlueRow, IEquatable { +public class MbpVariableDepthRow : IBlueRow, IEquatable { static List? schemaByNumLevels; public MbpVariableDepthRow(int numLevels) { @@ -32,7 +32,7 @@ public struct MbpVariableDepthRow : IBlueRow, IEquatable { public double[] AskQuantities { get; } public IBlueTableSchema schema { get; } - public readonly object? this[int index] { + public virtual object? this[int index] { get { switch (index) { case 0: @@ -91,8 +91,9 @@ public struct MbpVariableDepthRow : IBlueRow, IEquatable { } } - public bool Equals(MbpVariableDepthRow other) => - Id == other.Id + public bool Equals(MbpVariableDepthRow? other) => + other is not null + && Id == other.Id && Timestamp == other.Timestamp && Venue == other.Venue && Symbol == other.Symbol @@ -102,7 +103,7 @@ public struct MbpVariableDepthRow : IBlueRow, IEquatable { && AskQuantities.SequenceEqual(other.AskQuantities); public override bool Equals(object? obj) => - obj is MbpVariableDepthRow other && Equals(other); + Equals(obj as MbpVariableDepthRow); public override int GetHashCode() => Id; diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/OrderImbalanceStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/OrderImbalanceStreamFactory.cs index 4494f9bfe..eca954e5c 100644 --- a/src/DataStreamSuite/SharedStreams/StreamFactories/OrderImbalanceStreamFactory.cs +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/OrderImbalanceStreamFactory.cs @@ -7,13 +7,12 @@ using bts.SilverPine.StreamFactory.Logging; using bts.SilverPine.Tables; using Bts.DataStream; - -using mbts.utils; +using Bts.DataStreamCore; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using System.Reactive.Linq; +using Spark.Core; public class OrderImbalanceStreamFactory : IDataStreamFactory { readonly DatabaseConfig config; @@ -27,81 +26,57 @@ public class OrderImbalanceStreamFactory : IDataStreamFactory { this.logger = logger; } + public static IObservable>? CreateOrderImbalanceTables( + DatabaseConfig config, OrderImbalanceStreamParams args, ILogger logger) { + args.Validate(); + return new HistoricDataLoader(config, logger).LoadTablesObservable( + new HistoricDataParams() { + DataType = "mbp10", + Venue = args.Venue, + Symbol = args.Symbol, + BeginDate = args.BeginDate, + EndDate = args.EndDate, + Interval = args.Interval, + TsNanosCol = "ts_event_ns" + }, + RowFactory); + + OrderImbalanceRow RowFactory(IBlueRow r, int id, DateTime ts) { + var mbpRow = new OrderImbalanceRow() { + Id = r.Id(), + Timestamp = ts, + Venue = args.Venue, + Symbol = args.Symbol, + }; + for (int i = 0; i < args.NumLevels; i++) { + mbpRow.BidQuantities[i] = r.GetFloat64($"bid_sz_{i:00}") ?? 0; + mbpRow.BidPrices[i] = mbpRow.BidQuantities[i] > 0 ? r.GetFloat64($"bid_px_{i:00}").assert_value() : double.NaN; + mbpRow.AskQuantities[i] = r.GetFloat64($"ask_sz_{i:00}") ?? 0; + mbpRow.AskPrices[i] = mbpRow.AskQuantities[i] > 0 ? r.GetFloat64($"ask_px_{i:00}").assert_value() : double.NaN; + } + return mbpRow; + } + } + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { if (SupportedStreams.OrderImbalance.IsMatch(streamId)) { - return create(args); + return createOrderImbalance(args); } return null; } - ViewStream? create(IEnumerable<(string? Name, int? Index, object? Value)> args) { - var argStore = new StreamFactoryArgStore(args); - var venue = argStore.GetVenueString(); - if (venue is null) { - logger.LogMissingParam("orderImbalance", "venue"); - return null; - } - var symbol = argStore.GetSymbol(); - if (symbol is null) { - logger.LogMissingParam("orderImbalance", "symbol"); - return null; - } - var begin = argStore.GetDateTime("begin"); - if (!begin.HasValue) { - logger.LogMissingParam("orderImbalance", "begin"); - return null; - } - var end = argStore.GetDateTime("end"); - if (!end.HasValue) { - logger.LogMissingParam("orderImbalance", "end"); - return null; - } - var interval = argStore.GetTimeSpan("interval"); - if (!interval.HasValue) { - var totalDur = end.Value - begin.Value; - var targetInterval = totalDur / 100; - if (targetInterval.TotalDays > 5) { - interval = TimeSpan.FromDays(1); - } else if (targetInterval.TotalDays > 1) { - interval = TimeSpan.FromHours(1); - } else if (targetInterval.TotalMinutes > 60) { - interval = TimeSpan.FromMinutes(1); - } else if (targetInterval.TotalMinutes > 10) { - interval = TimeSpan.FromSeconds(1); - } else { - logger.LogWarning("OrderImbalanceStream interval wasn't specified. Defaulting to 1 second"); - interval = TimeSpan.FromSeconds(1); - } - // logger.LogMissingParam("orderImbalance", "interval"); - // return null; - } - var numLevels = argStore.GetInt32("levels") ?? 5; - var mbpSource = MbpStreamFactory.CreateTables(config, venue, symbol, begin.Value, end.Value, interval.Value, numLevels); - if (mbpSource is null) { - return null; - } - var imbalanceSource = mbpSource.SelectKeyValue( - r => new OrderImbalanceRow() { - Id = r.Id, - Timestamp = r.Timestamp, - Venue = r.Venue, - Symbol = r.Symbol, - // BidPrice = r.BidPrices.TakeWhile(p => !double.IsNaN(p)).LastOrDefault(double.NaN), - // BidQuantity = r.BidQuantities.TakeWhile(q => q > 0).Sum(), - // AskPrice = r.BidPrices.TakeWhile(p => !double.IsNaN(p)).LastOrDefault(double.NaN), - // AskQuantity = r.BidQuantities.TakeWhile(q => q > 0).Sum(), - }, - r => r.Id); - var vs = imbalanceSource - ?.AsDataPackets(MbpRow.Schema) - ?.AsViewStream(); - return vs; - } - public IEnumerable GetSupportedStreams() { yield return SupportedStreams.OrderImbalance; } + ViewStream? createOrderImbalance(IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (!StreamFactoryArgStore.TryGetParams(args, out OrderImbalanceStreamParams? sp, out var err)) { + logger.LogInvalidParams("orderImbalance", err); + return null; + } + return CreateOrderImbalanceTables(config, sp, logger)?.AsViewStream(BboRow.Schema); + } + static class SupportedStreams { public static readonly DataStreamMetadata OrderImbalance = new() { Name = OrderImbalanceRow.Schema.tableName diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/OrderImbalanceStreamParams.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/OrderImbalanceStreamParams.cs new file mode 100644 index 000000000..7305c3a8f --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/OrderImbalanceStreamParams.cs @@ -0,0 +1,37 @@ +namespace bts.SilverPine; + +using System.Diagnostics.CodeAnalysis; + +public class OrderImbalanceStreamParams { + [StreamArgument("venue", "exchange", "exch", IsRequired = true)] + public string? Venue { get; set; } + + [StreamArgument("symbol", "sym", "product", "asset", IsRequired = true)] + public string? Symbol { get; set; } + + [StreamArgument("begin", "beginDate", "start", "startDate", "from", IsRequired = true)] + public DateTime BeginDate { get; set; } + + [StreamArgument("end", "endDate", "stop", "stopDate", "to", IsRequired = true)] + public DateTime EndDate { get; set; } + + [StreamArgument("interval", IsRequired = true)] + public TimeSpan Interval { get; set; } + + [StreamArgument("levels")] + public int NumLevels { get; set; } + + [MemberNotNull(nameof(Venue), nameof(Symbol))] + public OrderImbalanceStreamParams Validate() { + if (Venue is null) { + throw new InvalidOperationException("Venue is null"); + } + if (Symbol is null) { + throw new InvalidOperationException("Symbol is null"); + } + if (Interval <= TimeSpan.Zero) { + throw new InvalidOperationException("Interval <= TimeSpan.Zero"); + } + return this; + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/OrderLatencyStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/OrderLatencyStreamFactory.cs new file mode 100644 index 000000000..ec52db7ec --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/OrderLatencyStreamFactory.cs @@ -0,0 +1,86 @@ +namespace bts.SilverPine; + +using BCG; + +using bts.api.core; +using bts.SilverPine.StreamFactory.Logging; +using bts.SilverPine.Tables; + +using Bts.DataStream; +using Bts.DataStreamCore; + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using Spark.Core; + +public class OrderLatencyStreamFactory : IDataStreamFactory { + readonly DatabaseConfig config; + readonly ILogger logger; + + public OrderLatencyStreamFactory(IOptions config, ILogger logger) + : this(config.Value, logger) { } + + public OrderLatencyStreamFactory(DatabaseConfig config, ILogger logger) { + this.config = config; + this.logger = logger; + } + + public static IObservable>? CreateOrderLatencyTables( + DatabaseConfig config, OrderLatencyStreamParams args, ILogger logger) { + args.Validate(); + int[]? colIndices = null; + return new HistoricDataLoader(config, logger).LoadTablesObservable( + new HistoricDataParams { + DataType = "mbo", + Venue = args.Venue, + Symbol = args.Symbol, + BeginDate = args.BeginDate, + EndDate = args.EndDate, + Interval = args.Interval, + TsNanosCol = "ts_recv_ns" + }, + RowFactory); + + OrderLatencyRow RowFactory(IBlueRow r, int id, DateTime ts) { + colIndices ??= r.schema.ResolveIndices(new[] { "sequence", "ts_event_ns", "price", "size", "flags" }); + var olRow = new OrderLatencyRow() { + Id = r.Id(), + Timestamp = ts, + Venue = args.Venue, + Symbol = args.Symbol, + SequenceNumber = r.GetInt32(colIndices[0]).assert_value(), + SendTimestamp = r.GetUnixEpochDateTime(colIndices[1]).assert_value(), + Price = r.GetFloat64(colIndices[2]).assert_value(), + Quantity = r.GetFloat64(colIndices[3]).assert_value(), + Flags = r.get(colIndices[4]), + }; + return olRow; + } + } + + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (SupportedStreams.OrderLatency.IsMatch(streamId)) { + return createOrderLatency(args); + } + return null; + } + + public IEnumerable GetSupportedStreams() { + yield return SupportedStreams.OrderLatency; + } + + ViewStream? createOrderLatency(IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (!StreamFactoryArgStore.TryGetParams(args, out OrderLatencyStreamParams? sp, out var err)) { + logger.LogInvalidParams("orderLatency", err); + return null; + } + return CreateOrderLatencyTables(config, sp, logger)?.AsViewStream(OrderLatencyRow.Schema); + } + + static class SupportedStreams { + public static readonly DataStreamMetadata OrderLatency = new() { + Name = OrderLatencyRow.Schema.tableName + }; + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/OrderLatencyStreamParams.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/OrderLatencyStreamParams.cs new file mode 100644 index 000000000..cc3a206c7 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/OrderLatencyStreamParams.cs @@ -0,0 +1,34 @@ +namespace bts.SilverPine; + +using System.Diagnostics.CodeAnalysis; + +public class OrderLatencyStreamParams { + [StreamArgument("venue", "exchange", "exch", IsRequired = true)] + public string? Venue { get; set; } + + [StreamArgument("symbol", "sym", "product", "asset", IsRequired = true)] + public string? Symbol { get; set; } + + [StreamArgument("begin", "beginDate", "start", "startDate", "from", IsRequired = true)] + public DateTime BeginDate { get; set; } + + [StreamArgument("end", "endDate", "stop", "stopDate", "to", IsRequired = true)] + public DateTime EndDate { get; set; } + + [StreamArgument("interval", IsRequired = true)] + public TimeSpan Interval { get; set; } = TimeSpan.Zero; + + [MemberNotNull(nameof(Venue), nameof(Symbol))] + public OrderLatencyStreamParams Validate() { + if (Venue is null) { + throw new InvalidOperationException("Venue is null"); + } + if (Symbol is null) { + throw new InvalidOperationException("Symbol is null"); + } + if (Interval < TimeSpan.Zero) { + throw new InvalidOperationException("Interval < TimeSpan.Zero"); + } + return this; + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/OrderLifespanStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/OrderLifespanStreamFactory.cs new file mode 100644 index 000000000..2b7877e7e --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/OrderLifespanStreamFactory.cs @@ -0,0 +1,108 @@ +namespace bts.SilverPine; + +using BCG; + +using bts.api.core; +using bts.SilverPine.StreamFactory.Logging; +using bts.SilverPine.Tables; + +using Bts.DataStream; +using Bts.DataStreamCore; + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using System.Reactive.Linq; + +public class OrderLifespanStreamFactory : IDataStreamFactory { + readonly DatabaseConfig config; + readonly ILogger logger; + + public OrderLifespanStreamFactory(IOptions config, ILogger logger) + : this(config.Value, logger) { } + + public OrderLifespanStreamFactory(DatabaseConfig config, ILogger logger) { + this.config = config; + this.logger = logger; + } + + public IObservable>? CreateOrderLifespanTables( + OrderLifespanStreamParams args) { + args.Validate(); + return MboStreamFactory.CreateTables(config, args.Venue, args.Symbol, args.BeginDate, args.EndDate) + .GroupTable(r => r.Id(), r => r.OrderId) + .BufferTable(TimeSpan.FromMilliseconds(500)) + .Scan(TableUpdate.mk_empty(r => r.Id), Aggregate); + + static TableUpdate Aggregate( + TableUpdate res, + TableUpdate, int> groups + ) { + var bd = res.to_builder(); + foreach (var g in groups.added.Concat(groups.changed)) { + var r = RowFactory(g); + if (r is null) { + bd.remove(g.group_key); + continue; + } + bd.add_or_update(r); + } + return bd.to_table(); + } + + static OrderLifespanRow? RowFactory(TableUpdateGrouping group) { + var rows = group.table.state.Values.OrderBy(r => r.Timestamp).ToList(); + if (rows.Count < 2) { + return null; + } + if (rows[0].Action != "A") { + return null; + } + if (rows.Any(r => r.Action == "M")) { + return null; + } + var fills = rows.Where(r => r.Action == "F").ToList(); + var filledQty = fills.Sum(r => r.Quantity); + var state = filledQty == rows[0].Quantity ? "Filled" : "Canceled"; + var rv = new OrderLifespanRow() { + Id = rows[0].OrderId, + Timestamp = rows[0].Timestamp, + SendTimestamp = rows[0].SendTimestamp, + Venue = rows[0].Venue, + Symbol = rows[0].Symbol, + Side = rows[0].Side, + Price = rows[0].Price, + Quantity = rows[0].Quantity, + FinalTimestamp = rows[^1].Timestamp, + FilledQuantity = filledQty, + State = state, + }; + return rv; + } + } + + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (SupportedStreams.OrderLifespan.IsMatch(streamId)) { + return createOrderLifespan(args); + } + return null; + } + + public IEnumerable GetSupportedStreams() { + yield return SupportedStreams.OrderLifespan; + } + + ViewStream? createOrderLifespan(IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (!StreamFactoryArgStore.TryGetParams(args, out OrderLifespanStreamParams? sp, out var err)) { + logger.LogInvalidParams("orderLifespan", err); + return null; + } + return CreateOrderLifespanTables(sp)?.AsViewStream(OrderLifespanRow.Schema); + } + + static class SupportedStreams { + public static readonly DataStreamMetadata OrderLifespan = new() { + Name = OrderLifespanRow.Schema.tableName + }; + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/OrderLifespanStreamParams.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/OrderLifespanStreamParams.cs new file mode 100644 index 000000000..7adaa5764 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/OrderLifespanStreamParams.cs @@ -0,0 +1,37 @@ +namespace bts.SilverPine; + +using System.Diagnostics.CodeAnalysis; + +public class OrderLifespanStreamParams { + [StreamArgument("venue", "exchange", "exch", IsRequired = true)] + public string? Venue { get; set; } + + [StreamArgument("symbol", "sym", "product", "asset", IsRequired = true)] + public string? Symbol { get; set; } + + [StreamArgument("begin", "beginDate", "start", "startDate", "from", IsRequired = true)] + public DateTime BeginDate { get; set; } + + [StreamArgument("end", "endDate", "stop", "stopDate", "to", IsRequired = true)] + public DateTime EndDate { get; set; } + + [StreamArgument("interval", IsRequired = true)] + public TimeSpan Interval { get; set; } + + [StreamArgument("levels")] + public int NumLevels { get; set; } + + [MemberNotNull(nameof(Venue), nameof(Symbol))] + public OrderLifespanStreamParams Validate() { + if (Venue is null) { + throw new InvalidOperationException("Venue is null"); + } + if (Symbol is null) { + throw new InvalidOperationException("Symbol is null"); + } + if (Interval < TimeSpan.Zero) { + throw new InvalidOperationException("Interval < TimeSpan.Zero"); + } + return this; + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/StreamFactoryArgStore.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/StreamFactoryArgStore.cs index 8b8038507..0576f2079 100644 --- a/src/DataStreamSuite/SharedStreams/StreamFactories/StreamFactoryArgStore.cs +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/StreamFactoryArgStore.cs @@ -1,9 +1,14 @@ namespace bts.SilverPine; +using BCG; + using bts.api; +using bts.SilverPine.StreamFactory.Logging; using Microsoft.Extensions.Logging; +using Spark.Core; + using System.Diagnostics.CodeAnalysis; using System.Reflection; @@ -11,9 +16,14 @@ using System.Text; class StreamFactoryArgStore { readonly IEnumerable<(string? Name, int? Index, object? Value)> args; + readonly ILogger? logger; + readonly string streamType; - public StreamFactoryArgStore(IEnumerable<(string? Name, int? Index, object? Value)> args) => + public StreamFactoryArgStore(IEnumerable<(string? Name, int? Index, object? Value)> args, string? streamType = null, ILogger? logger = null) { this.args = args; + this.streamType = streamType ?? "anonStream"; + this.logger = logger; + } public static bool TryGetParams( IEnumerable<(string? Name, int? Index, object? Value)> args, @@ -22,7 +32,7 @@ class StreamFactoryArgStore { ) => new StreamFactoryArgStore(args).TryGetParams(out result, out errorMessage); - public bool TryGetParams([MaybeNull][NotNullWhen(true)]out T result, [NotNullWhen(false)]out string? errorMessage) { + public bool TryGetParams([MaybeNull][NotNullWhen(true)] out T result, [NotNullWhen(false)] out string? errorMessage) { var props = typeof(T).GetProperties(); var res = Activator.CreateInstance(); if (res is null) { @@ -32,8 +42,17 @@ class StreamFactoryArgStore { } foreach (var p in props) { var att = p.GetCustomAttribute(); + if (p.PropertyType == typeof(DateTime)) { + + } if (TryGet(att?.Names ?? new[] { p.Name }, out var val)) { - p.SetValue(res, val); + if (p.PropertyType == typeof(DateTime)) { + p.SetValue(res, BlueDataUtility.GetDateTime(val).assert_value()); + } else if (p.PropertyType == typeof(TimeSpan)) { + p.SetValue(res, BlueDataUtility.get_timespan(val).assert_value()); + } else { + p.SetValue(res, val); + } } else if (att?.IsRequired ?? false) { errorMessage = $"Missing required param '{p.Name}'"; result = default; @@ -45,8 +64,13 @@ class StreamFactoryArgStore { return true; } - public Venue GetVenue() => - GetVenue(new[] { "venue", "exchange", "exch" }); + public Venue GetVenue(bool required = true) { + var rv = GetVenue(new[] { "venue", "exchange", "exch" }); + if (required && logger is not null && rv == Venue.Unset) { + logger.LogMissingParam(streamType, "venue"); + } + return rv; + } public Venue GetVenue(string[] preferredNames) { return preferredNames @@ -68,13 +92,22 @@ class StreamFactoryArgStore { } } - public string? GetVenueString() => - GetVenueString("venue", "exchange", "exch"); + public string? GetVenueString(bool required = true) => + GetVenueString(required, "venue", "exchange", "exch"); - public string? GetVenueString(params string[] preferredNames) { + public bool GetVenueString([NotNullWhen(true)] out string? venue, bool required = true) { + venue = GetVenueString(required, "venue", "exchange", "exch"); + return venue is not null; + } + + public string? GetVenueString(bool required, params string[] preferredNames) { var obj = Get(preferredNames); var str = obj as string ?? obj?.ToString(); - return str?.ToUpper()?.Replace("VENUE_", ""); + str = str?.ToUpper()?.Replace("VENUE_", ""); + if (required && logger is not null && string.IsNullOrEmpty(str)) { + logger.LogMissingParam(streamType, "venue"); + } + return str; } public SecurityType GetSecurityType() => @@ -100,22 +133,57 @@ class StreamFactoryArgStore { } } - public string? GetSymbol() => - Get("symbol", "product", "productCode", "asset")?.ToString(); + public bool GetSymbol([NotNullWhen(true)] out string? symbol, bool required = true) { + symbol = GetSymbol(required); + return symbol is not null; + } - public int? GetInt32(params string[] names) { + public string? GetSymbol(bool required = true) { + var rv = Get("symbol", "product", "productCode", "asset")?.ToString(); + if (required && logger is not null && string.IsNullOrEmpty(rv)) { + logger.LogMissingParam(streamType, "symbol"); + } + return rv; + } + + public bool GetInt32(string name, [NotNullWhen(true)] out int? result) => + GetInt32(out result, true, name); + + public bool GetInt32([NotNullWhen(true)] out int? result, bool required, params string[] names) { + result = GetInt32(names); + return result.HasValue; + } + + public int? GetInt32(params string[] names) => + GetInt32(true, names); + + public int? GetInt32(bool required, params string[] names) { + if (names.Length == 0) { + return null; + } var obj = Get(names); if (obj is int i) { return i; } var s = obj as string ?? obj?.ToString(); - return int.TryParse(s, out i) ? i : null; + if (int.TryParse(s, out i)) { + return i; + } + if (required && logger is not null) { + logger.LogMissingParam(streamType, names[0]); + } + return null; + } + + public bool GetTimeSpan(string name, out TimeSpan? result) { + result = GetTimeSpan(true, new[] { name }); + return result.HasValue; } public TimeSpan? GetTimeSpan(string name) => - GetTimeSpan(new[] { name }); + GetTimeSpan(true, new[] { name }); - public TimeSpan? GetTimeSpan(string[] preferredNames) => + public TimeSpan? GetTimeSpan(bool required, string[] preferredNames) => preferredNames .Select(n => BCG.BlueDataUtility.get_timespan(Get(n))) .Where(ts => ts.HasValue) @@ -142,11 +210,34 @@ class StreamFactoryArgStore { public IEnumerable GetDateTimes() => args.Select(a => convertToDateTime(a.Value)).SelectWhereValues(); + public bool GetDateTime(string name, [NotNullWhen(true)] out DateTime? result) { + result = GetDateTime(true, name); + return result.HasValue; + } + + public bool GetDateTime([NotNullWhen(true)] out DateTime? result, params string[] names) { + result = GetDateTime(names); + return result.HasValue; + } + public DateTime? GetDateTime(params string[] names) => + GetDateTime(true, names); + + public DateTime? GetDateTime(bool required, params string[] names) => names.Select(n => Get(n)).Select(convertToDateTime).Where(dt => dt.HasValue).FirstOrDefault(); public object? Get(params string[] names) => - TryGet(names, out var result) ? result : null; + Get(true, names); + + public object? Get(bool required, params string[] names) { + if (names.Length == 0) { + return null; + } + if (!TryGet(names, out var result) && required && logger is not null) { + logger.LogMissingParam(streamType, names[0]); + } + return result; + } public bool TryGet(string name, out object? result) => TryGet(new[] { name }, out result); @@ -230,8 +321,20 @@ class StreamFactoryArgStore { return true; } if (obj is string s) { + // if (OffsetDateTimePattern) + if (DateTimeOffset.TryParseExact(s, "yyyy-MM-ddTHH:mm:ssK", null, System.Globalization.DateTimeStyles.None, out var dto)) { + res = dto.ToUniversalTime().DateTime; + return true; + } + if (DateTimeOffset.TryParseExact(s, "yyyy-MM-ddTHH:mm:ss.fffffffffK", null, System.Globalization.DateTimeStyles.None, out dto)) { + res = dto.ToUniversalTime().DateTime; + return true; + } + if (DateTime.TryParse(s, null, System.Globalization.DateTimeStyles.RoundtripKind, out res)) { + return true; + } if (DateTimeOffset.TryParse(s, out var resOff)) { - res = resOff.DateTime; + res = resOff.ToUniversalTime().DateTime; return true; } return DateTime.TryParse(s, out res);