diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index ceeb06ae8..02a9e91bd 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -25,6 +25,12 @@ "commands": [ "fake" ] + }, + "dotnet-t4": { + "version": "3.0.0", + "commands": [ + "t4" + ] } } } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Misc/DSLStream.cs b/src/DataStreamSuite/DataStream/Misc/DSLStream.cs index dea082576..5ed3fdbcf 100644 --- a/src/DataStreamSuite/DataStream/Misc/DSLStream.cs +++ b/src/DataStreamSuite/DataStream/Misc/DSLStream.cs @@ -3,23 +3,24 @@ namespace bts.BlueOrb { using BCG; using System.Collections.Generic; using Bts.DataStreamExpressions; + using Bts.DataStreamCore; // All this does is take in a dsl expression, parse it and fire the data that come out of running it. // So for example, if an expresison is passed [DataStreamUserRespresentation("dsl")] public class DSLStream : ViewStream { private readonly string _dsl; - public DSLStream(string dsl) { + public DSLStream(string dsl, IDataStreamFactory streamFactory) { Log.Gen.info($"DSLStream.cons dsl: {dsl}"); _dsl = dsl; - ViewStream rs = build_stream(dsl) ?? new LiteralStream("error", $"msg:invalid dsl"); + var rs = build_stream(streamFactory, dsl) ?? new LiteralStream("error", $"msg:invalid dsl"); mySetRealSource(rs); } - static ViewStream? build_stream(string stream_expression) { + static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) { try { var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); - var rv = exp.getView(new Dictionary(0)); + var rv = exp.getView(new Dictionary(0), streamFactory); return rv; } catch (Exception ex) { diff --git a/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs b/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs index be8b833a2..d129236f0 100644 --- a/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs +++ b/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs @@ -35,12 +35,12 @@ namespace BCG { view_publisher = view_publisher_; } } - static ViewStream? build_stream(string stream_expression) + static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) { try { var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); - var rv = exp.getView(new Dictionary(0)); + var rv = exp.getView(new Dictionary(0), streamFactory); return rv; } catch (Exception ex) { @@ -52,15 +52,17 @@ namespace BCG { _stream_pub_map = new Dictionary(); readonly IPubTableBroker _broker; readonly IManagedSubscriber _sub; + readonly IDataStreamFactory streamFactory; readonly object syncRoot = new(); IDisposable? cloudStreamsSub; IDisposable? remoteRequestsSub; bool isDisposed; - public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) { + public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker, IDataStreamFactory streamFactory) { _sub = managedSubscriber; _broker = pubTableBroker; + this.streamFactory = streamFactory; } public Task StartAsync(CancellationToken cancellationToken) { @@ -103,7 +105,11 @@ namespace BCG { private void _spawn_remote_stream(string name, string stream_expression, bool persist) { bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}"); - ViewStream stream = build_stream(stream_expression); + var stream = build_stream(streamFactory, stream_expression); + if (stream is null) { + Log.DataFlow.error($"Failed to spawn remote stream. name: {name}; expression: {stream_expression}"); + return; + } _expose_to_ws(name, stream); if (persist) { string tempDir = Path.GetTempPath(); diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs index 71e4fccd7..2308cea08 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs @@ -19,6 +19,8 @@ using Newtonsoft.Json; using BCG; using System.Runtime.CompilerServices; using bts; +using Spark.Core; +using System.Numerics; namespace bts { @@ -91,13 +93,13 @@ namespace BCG { return BlueDataUtility.get_row_datetime_value(r, col_no, err_value); } - public static double? GetDouble(this IBlueRow r, int colNo) => - GetDouble(r[colNo]); + public static double? GetFloat64(this IBlueRow r, int colNo) => + GetFloat64(r[colNo]); - public static double? GetDouble(this IBlueRow r, string colName) => - GetDouble(r[colName]); + public static double? GetFloat64(this IBlueRow r, string colName) => + GetFloat64(r[colName]); - public static double? GetDouble(object obj) { + public static double? GetFloat64(object obj) { if (obj is double d) { return d; } @@ -125,6 +127,133 @@ namespace BCG { return null; } + public static int? GetInt32(this IBlueRow r, int colNo) => + GetInt32(r[colNo]); + + public static int? GetInt32(this IBlueRow r, string colName) => + GetInt32(r[colName]); + + public static int? GetInt32(object obj) { + checked { + if (obj is double d) { + return (int)d; + } + if (obj is int i32) { + return i32; + } + if (obj is long i64) { + return (int)i64; + } + if (obj is string s) { + return double.TryParse(s, out d) ? (int)d : null; + } + if (obj is uint u32) { + return (int)u32; + } + if (obj is ulong u64) { + return (int)u64; + } + if (obj is short i16) { + return i16; + } + if (obj is ushort u16) { + return u16; + } + } + return null; + } + + public static long? GetInt64(this IBlueRow r, int colNo) => + GetInt64(r[colNo]); + + public static long? GetInt64(this IBlueRow r, string colName) => + GetInt64(r[colName]); + + public static long? GetInt64(object obj) { + checked { + if (obj is double d) { + return (long)d; + } + if (obj is int i32) { + return i32; + } + if (obj is long i64) { + return i64; + } + if (obj is string s) { + return double.TryParse(s, out d) ? (long)d : null; + } + if (obj is uint u32) { + return u32; + } + if (obj is ulong u64) { + return (long)u64; + } + if (obj is short i16) { + return i16; + } + if (obj is ushort u16) { + return u16; + } + } + return null; + } + + public static DateTime? GetUnixEpochDateTime(this IBlueRow row, string colName) => + GetUnixEpochDateTime(row[colName]); + + public static DateTime? GetUnixEpochDateTime(this IBlueRow row, int colNo) => + GetUnixEpochDateTime(row[colNo]); + + public static DateTime? GetUnixEpochDateTime(object obj) { + var tsNanos = GetInt64(obj); + return tsNanos.HasValue ? DateTime.UnixEpoch.Add(new TimeSpan(tsNanos.Value / 100)) : null; + } + + public static DateTime? GetDateTime(this IBlueRow row, string colName) => + GetDateTime(row[colName]); + + public static DateTime? GetDateTime(object obj) { + var res = _get_datetime_value(obj, DateTime.MinValue); + if (res == DateTime.MinValue) { + return null; + } + return res; + } + + public static object GetEnum(Type type, object o) { + try { + if (o is null || o == DBNull.Value) { + return null; + } else if (o is string str) { + return Enum.TryParse(type, str, true, out var res) ? res : null; + } else if (o is int || o is uint || o is long || o is ulong || o is short || o is ushort) { + return Convert.ChangeType(o, type); + } + } catch { + } + return null; + } + + public static T? GetEnum(this IBlueRow row, string colName) where T : struct, Enum => + GetEnum(row[colName]); + + public static T? GetEnum(object o) where T : struct, Enum { + try { + if (o is null || o == DBNull.Value) { + return null; + } else if (o is string str) { + return Enum.TryParse(str, true, out T res) ? res : null; + } else if (o is int || o is short || o is ushort || o is uint || o is long || o is ulong) + return (T)o; + else if (o is T t) { + return t; + } + } catch { + } + return null; + } + public static T get_enum(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct { return BlueDataUtility.get_row_enum_value(r, col_name, err_value); } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowStore.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowStore.cs new file mode 100644 index 000000000..28ab1b035 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowStore.cs @@ -0,0 +1,311 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using mbts.utils; + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; + +public class BlueRowStore : IBlueRow, IReadOnlyDictionary { + readonly string? name; + IBlueTableSchema? _schema; + + public BlueRowStore() { } + + public BlueRowStore( + int id, + (string Name, T1 Value) item1, + (string Name, T2 Value) item2, + (string Name, T3 Value) item3, + (string Name, T4 Value) item4, + (string Name, T5 Value) item5, + (string Name, T6 Value) item6 + ) { + Id = id; + Item1 = item1; + Item2 = item2; + Item3 = item3; + Item4 = item4; + Item5 = item5; + Item6 = item6; + } + + public int Id { get; set; } + public (string Name, T1 Value) Item1 { get; set; } + public (string Name, T2 Value) Item2 { get; set; } + public (string Name, T3 Value) Item3 { get; set; } + public (string Name, T4 Value) Item4 { get; set; } + public (string Name, T5 Value) Item5 { get; set; } + public (string Name, T6 Value) Item6 { get; set; } + + public int Count => 5; + public IReadOnlyList Keys => new KeyList(this); + public IReadOnlyList Values => new ValueList(this); + + public IBlueTableSchema schema => + _schema ??= new BlueTableSchema(name, new ColumnInfo[]{ + new("id", 0, typeof(int)), + new(Item1.Name, 1, typeof(T1)), + new(Item2.Name, 2, typeof(T2)), + new(Item3.Name, 3, typeof(T3)), + new(Item4.Name, 4, typeof(T4)), + new(Item5.Name, 5, typeof(T5)), + new(Item6.Name, 6, typeof(T6)), + }); + + ICollection IDictionary.Keys => new KeyList(this); + ICollection IDictionary.Values => new ValueList(this); + + IEnumerable IReadOnlyDictionary.Keys => Keys; + IEnumerable IReadOnlyDictionary.Values => Values; + + bool ICollection.IsSynchronized => false; + object ICollection.SyncRoot => this; + + public object? this[string colName] { + get { + if (colName == "id") { + return Id; + } + if (colName == Item1.Name) { + return Item1.Value; + } + if (colName == Item2.Name) { + return Item2.Value; + } + if (colName == Item3.Name) { + return Item3.Value; + } + if (colName == Item4.Name) { + return Item4.Value; + } + if (colName == Item5.Name) { + return Item5.Value; + } + if (colName == Item6.Name) { + return Item6.Value; + } + throw new KeyNotFoundException(); + } + set { + if (colName == "id") { + Id = ExceptionAssert.Cast(value).To(); + } + if (colName == Item1.Name) { + Item1 = (Item1.Name, ExceptionAssert.Cast(value).To()); + } + if (colName == Item2.Name) { + Item2 = (Item2.Name, ExceptionAssert.Cast(value).To()); + } + if (colName == Item3.Name) { + Item3 = (Item3.Name, ExceptionAssert.Cast(value).To()); + } + if (colName == Item4.Name) { + Item4 = (Item4.Name, ExceptionAssert.Cast(value).To()); + } + if (colName == Item5.Name) { + Item5 = (Item5.Name, ExceptionAssert.Cast(value).To()); + } + if (colName == Item6.Name) { + Item6 = (Item6.Name, ExceptionAssert.Cast(value).To()); + } + throw new KeyNotFoundException(); + } + } + + public object? this[int index] => index switch { + 0 => Id, + 1 => Item1.Value, + 2 => Item2.Value, + 3 => Item3.Value, + 4 => Item4.Value, + 5 => Item5.Value, + 6 => Item6.Value, + _ => throw new IndexOutOfRangeException() + }; + + object? IDictionary.this[object key] { + get => this[ExceptionAssert.Cast(key).To()]; + set => this[ExceptionAssert.Cast(key).To()] = value; + } + + public void CopyTo(Array array, int index) { + array.SetValue(Id, index++); + array.SetValue(Item1.Value, index++); + array.SetValue(Item2.Value, index++); + array.SetValue(Item3.Value, index++); + array.SetValue(Item4.Value, index++); + array.SetValue(Item5.Value, index++); + array.SetValue(Item6.Value, index++); + } + + public bool Contains(object key) => + key is string keystr && ContainsKey(keystr); + + public bool ContainsKey(string key) => + Keys.Contains(key); + + public void SetId(int id) => + Id = id; + + public void SetValue1(T1 value) => + Item1 = (Item1.Name, value); + + public void SetValue1(string colName, T1 value) => + Item1 = (colName, value); + + public void SetValue2(T2 value) => + Item2 = (Item2.Name, value); + + public void SetValue2(string colName, T2 value) => + Item2 = (colName, value); + + public void SetValue3(T3 value) => + Item3 = (Item3.Name, value); + + public void SetValue3(string colName, T3 value) => + Item3 = (colName, value); + + public void SetValue4(T4 value) => + Item4 = (Item4.Name, value); + + public void SetValue4(string colName, T4 value) => + Item4 = (colName, value); + + public void SetValue5(T5 value) => + Item5 = (Item5.Name, value); + + public void SetValue5(string colName, T5 value) => + Item5 = (colName, value); + + public void SetValue6(T6 value) => + Item6 = (Item6.Name, value); + + public void SetValue6(string colName, T6 value) => + Item6 = (colName, value); + + public bool TryGetValue(string key, [MaybeNullWhen(false)] out object? value) { + if (!Keys.TryFindIndex(key, out int index)) { + value = null; + return false; + } + value = this[index]; + return true; + } + + void IDictionary.Add(object key, object? value) => + throw new NotSupportedException(); + + void IDictionary.Clear() => + throw new NotSupportedException(); + + void IDictionary.Remove(object key) => + throw new NotSupportedException(); + + IDictionaryEnumerator IDictionary.GetEnumerator() => + throw new NotImplementedException(); + + IEnumerator IEnumerable.GetEnumerator() => + throw new NotImplementedException(); + + public IEnumerator> GetEnumerator() { + throw new NotImplementedException(); + } + + class KeyList : ICollection, IReadOnlyList { + readonly BlueRowStore store; + + public KeyList(BlueRowStore store) => + this.store = store; + + public int Count => 5; + bool ICollection.IsSynchronized => false; + object ICollection.SyncRoot => this; + + public string this[int index] => + index switch { + 0 => "id", + 1 => store.Item1.Name, + 2 => store.Item2.Name, + 3 => store.Item3.Name, + 4 => store.Item4.Name, + 5 => store.Item5.Name, + 6 => store.Item6.Name, + _ => throw new IndexOutOfRangeException() + }; + + public void CopyTo(Array array, int index) { + array.SetValue(store.Id, index++); + array.SetValue(store.Item1.Value, index++); + array.SetValue(store.Item2.Value, index++); + array.SetValue(store.Item3.Value, index++); + array.SetValue(store.Item4.Value, index++); + array.SetValue(store.Item5.Value, index++); + array.SetValue(store.Item6.Value, index++); + } + + public IEnumerator GetEnumerator() { + yield return "id"; + yield return store.Item1.Name; + yield return store.Item2.Name; + yield return store.Item3.Name; + yield return store.Item4.Name; + yield return store.Item5.Name; + yield return store.Item6.Name; + } + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); + } + + class ValueList : ICollection, IReadOnlyList { + readonly BlueRowStore store; + + public ValueList(BlueRowStore store) => + this.store = store; + + public object? this[int index] => + index switch { + 0 => store.Id, + 1 => store.Item1.Value, + 2 => store.Item2.Value, + 3 => store.Item3.Value, + 4 => store.Item4.Value, + 5 => store.Item5.Value, + 6 => store.Item6.Value, + _ => throw new IndexOutOfRangeException() + }; + + public int Count => 5; + public bool IsSynchronized => false; + public object SyncRoot => this; + + public void CopyTo(Array array, int index) { + array.SetValue(store.Id, index++); + array.SetValue(store.Item1.Value, index++); + array.SetValue(store.Item2.Value, index++); + array.SetValue(store.Item3.Value, index++); + array.SetValue(store.Item4.Value, index++); + array.SetValue(store.Item5.Value, index++); + array.SetValue(store.Item6.Value, index++); + } + + public IEnumerator GetEnumerator() { + yield return store.Id; + yield return store.Item1.Value; + yield return store.Item2.Value; + yield return store.Item3.Value; + yield return store.Item4.Value; + yield return store.Item5.Value; + yield return store.Item6.Value; + } + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableListWrapper.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableListWrapper.cs new file mode 100644 index 000000000..1014acbe7 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableListWrapper.cs @@ -0,0 +1,49 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +public class BlueTableListWrapper : IBlueRowCollection { + readonly IBlueTableSchema schema; + readonly IList rows; + readonly Lazy> rowIndex; + + public BlueTableListWrapper(IBlueTableSchema schema, IList rows) { + this.schema = schema; + this.rows = rows; + rowIndex = new(() => rows.Select((r, i) => (i, r)).ToDictionary(r => r.r.Id())); + } + + public IBlueRow this[int id] => + rowIndex.Value.TryGetValue(id, out var r) + ? r.Row + : throw new KeyNotFoundException(); + + public IBlueTableSchema Schema => schema; + + IBlueTableSchema IBlueRowCollection.schema => schema; + + public int Count => rows.Count; + + public bool containsID(int id) => + rowIndex.Value.ContainsKey(id); + + public IEnumerator GetEnumerator() => + rows.GetEnumerator(); + + public IBlueRow GetRow(int index) => + rows[index]; + + public int IndexOf(IBlueRow item) => + rowIndex.Value.TryGetValue(item.Id(), out var r) && BlueRowEqualityComparer.Equals(r.Row, item) + ? r.Index + : -1; + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaExtensions.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaExtensions.cs new file mode 100644 index 000000000..745280109 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaExtensions.cs @@ -0,0 +1,87 @@ +namespace Bts.DataStreamCore; + +using BCG; + +using bts.collections; + +using System; + +public static class BlueTableSchemaExtensions { + public static int ResolveIndices(this IBlueTableSchema schema, string name) { + var indices = ResolveIndices(schema, new[] { name }); + return indices[0]; + } + + public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) { + var indices = ResolveIndices(schema, new[] { name1, name2 }); + return (indices[0], indices[1]); + } + + public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) { + var indices = ResolveIndices(schema, new[] { name1, name2, name3 }); + return (indices[0], indices[1], indices[2]); + } + + public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) { + var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 }); + return (indices[0], indices[1], indices[2], indices[3]); + } + + public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) => + ResolveIndices(schema.columnInfo, columnNames); + + public static int ResolveIndices(this ColumnInfo[] columns, string name) { + var indices = ResolveIndices(columns, new[] { name }); + return indices[0]; + } + + public static int[] ResolveIndices(this ColumnInfo[] columns, string[] columnNames) { + var indices = new int[columnNames.Length]; + int nResolved = 0; + for (int i = 0; i < columnNames.Length; i++) { + int origNResolved = nResolved; + for (int j = 0; j < columns.Length; j++) { + if (columns[j].colName == columnNames[i]) { + indices[i] = j; + nResolved++; + break; + } + } + if (nResolved == origNResolved) { + throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(columns)); + } + } + return indices; + } + + public static bool TryResolveIndices(this IBlueTableSchema schema, string name, out int result) { + var res = TryResolveIndices(schema, new[] { name })[0]; + result = res ?? -1; + return res.HasValue; + } + + public static int? TryResolveIndices(this IBlueTableSchema schema, string name) => + TryResolveIndices(schema, new[] { name })[0]; + + public static int? TryResolveIndices(this ColumnInfo[] columns, string name) => + TryResolveIndices(columns, new[] { name })[0]; + + public static int?[] TryResolveIndices(this IBlueTableSchema schema, string[] columnNames) => + TryResolveIndices(schema.columnInfo, columnNames); + + public static int?[] TryResolveIndices(this ColumnInfo[] columns, string[] columnNames) { + var indices = new int?[columnNames.Length]; + indices.Fill(null); + int nResolved = 0; + for (int i = 0; i < columnNames.Length; i++) { + for (int j = 0; j < columns.Length; j++) { + if (columns[j].colName == columnNames[i]) { + indices[i] = j; + nResolved++; + break; + } + } + } + return indices; + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/ITimeSeriesRow.cs b/src/DataStreamSuite/DataStreamCore/BlueData/ITimeSeriesRow.cs new file mode 100644 index 000000000..5a27bb3cf --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/ITimeSeriesRow.cs @@ -0,0 +1,12 @@ +namespace Bts.DataStreamCore; + +using BCG; + +using mbts.utils; + +using System; + +public interface ITimeSeriesRow : IBlueRow { + int Id => ExceptionAssert.of_type(this["id"]); + DateTime Timestamp { get; } +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/CompositeDataStreamFactory.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/CompositeDataStreamFactory.cs new file mode 100644 index 000000000..13294508f --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/CompositeDataStreamFactory.cs @@ -0,0 +1,27 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using System.Collections.Generic; +using System.Linq; + +public class CompositeDataStreamFactory : IDataStreamFactory { + readonly List streamFactories; + + public CompositeDataStreamFactory(IEnumerable streamFactories) { + this.streamFactories = streamFactories.ToList(); + foreach (var rsf in this.streamFactories.OfType()) { + rsf.OnCompositeFactory(this); + } + } + + public IEnumerable GetSupportedStreams() => + streamFactories.SelectMany(f => f.GetSupportedStreams()); + + public ViewStream? Create(string streamId, IEnumerable args) => + streamFactories.Where(f => f.CanCreate(streamId)) + .Select(f => f.Create(streamId, args)) + .ExceptNull() + .FirstOrDefault(); +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamFactory.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamFactory.cs new file mode 100644 index 000000000..23467f254 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamFactory.cs @@ -0,0 +1,220 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using Microsoft.Extensions.Logging; + +using System; +using System.Collections.Generic; +using System.Linq; + +public class DataStreamFactory : IDataStreamFactory { + readonly List factories = new(); + + public static IDataStreamFactory Create(IEnumerable streamFactories) { + var dsf = new DataStreamFactory(); + dsf.factories.AddRange(streamFactories); + return dsf.factories.Count == 1 ? dsf.factories[0] : dsf; + } + + public ViewStream? Create(string streamId, IEnumerable args) => + factories.Where(sf => sf.GetSupportedStreams().Any(m => m.IsMatch(streamId))) + .Select(sf => sf.Create(streamId, args)) + .ExceptNull() + .FirstOrDefault(); + + public IEnumerable GetSupportedStreams() => + factories.SelectMany(sf => sf.GetSupportedStreams()); + + public void Add(IDataStreamFactory streamFactory) => + factories.Add(streamFactory); +} + +public static class DataStreamFactoryExtensions { + public static void Add(this DataStreamFactory streamFactory, IDataStreamMetadata metadata, Func singleStreamfactory, ILogger? logger = null) { + var sf = SingleDataStreamFactory.FromDelegate(metadata, singleStreamfactory); + streamFactory.Add(sf); + } + + public static void Add(this DataStreamFactory streamFactory, IDataStreamMetadata metadata, Func, ViewStream?> singleStreamfactory, ILogger? logger = null) { + var sf = new SingleDataStreamFactory(metadata, singleStreamfactory); + streamFactory.Add(sf); + } + + public static void Add(this DataStreamFactory streamFactory, string name, Func, ViewStream?> singleStreamfactory, ILogger? logger = null) { + var sf = new SingleDataStreamFactory(new DataStreamMetadata() { Name = name }, singleStreamfactory); + streamFactory.Add(sf); + } + + public static void Add(this DataStreamFactory streamFactory, IDataStreamMetadata metadata, Func singleStreamfactory, ILogger? logger = null) { + var sf = new SingleDataStreamFactory(metadata, singleStreamfactory, logger); + streamFactory.Add(sf); + } + + public static void Add(this DataStreamFactory streamFactory, string name, Func singleStreamfactory) { + var sf = new SingleDataStreamFactory(new DataStreamMetadata(name), singleStreamfactory); + streamFactory.Add(sf); + } + + public static void Add(this DataStreamFactory streamFactory, string name, Func singleStreamfactory, ILogger? logger) { + var sf = new SingleDataStreamFactory(new DataStreamMetadata(name), singleStreamfactory, logger); + streamFactory.Add(sf); + } + + public static void Add(this DataStreamFactory streamFactory, string name, Func factory, IStreamArgumentParser? parser = null) { + var meta = new DataStreamMetadata() { Name = name }; + parser ??= StreamArgumentParser.Default(); + var sf = new SingleDataStreamFactory(meta, FactoryWrapper); + streamFactory.Add(sf); + + ViewStream? FactoryWrapper(IEnumerable args) { + if (!args.TryGetSingle(out var a)) { + return null; + } + if (!parser.CanParse(a, typeof(T))) { + return null; + } + if (!parser.TryParse(a, out var res, out var err)) { + return null; + } + return factory(res); + } + } + + public static void Add( + this DataStreamFactory streamFactory, + string name, + Func factory, + IStreamArgumentParser? parser1 = null, + IStreamArgumentParser? parser2 = null + ) { + var meta = new DataStreamMetadata() { Name = name }; + parser1 ??= StreamArgumentParser.Default(); + parser2 ??= StreamArgumentParser.Default(); + var sf = new SingleDataStreamFactory(meta, FactoryWrapper); + streamFactory.Add(sf); + + ViewStream? FactoryWrapper(IEnumerable args) { + if (!args.TryGetExactCount(out var a1, out var a2)) { + return null; + } + if (!parser1.CanAndTryParse(a2, out var v1, out var err1)) { + return null; + } + if (!parser2.CanAndTryParse(a2, out var v2, out var err2)) { + return null; + } + return factory(v1, v2); + } + } + + public static void Add( + this DataStreamFactory streamFactory, + string name, + Func factory, + IStreamArgumentParser? parser1 = null, + IStreamArgumentParser? parser2 = null, + IStreamArgumentParser? parser3 = null + ) { + var meta = new DataStreamMetadata() { Name = name }; + parser1 ??= StreamArgumentParser.Default(); + parser2 ??= StreamArgumentParser.Default(); + parser3 ??= StreamArgumentParser.Default(); + var sf = new SingleDataStreamFactory(meta, FactoryWrapper); + streamFactory.Add(sf); + + ViewStream? FactoryWrapper(IEnumerable args) { + if (!args.TryGetExactCount(out var a1, out var a2, out var a3)) { + return null; + } + if (!parser1.CanAndTryParse(a2, out var v1, out var err1)) { + return null; + } + if (!parser2.CanAndTryParse(a2, out var v2, out var err2)) { + return null; + } + if (!parser3.CanAndTryParse(a3, out var v3, out var err3)) { + return null; + } + return factory(v1, v2, v3); + } + } + + public static void Add( + this DataStreamFactory streamFactory, + string name, + Func factory, + IStreamArgumentParser? parser1 = null, + IStreamArgumentParser? parser2 = null, + IStreamArgumentParser? parser3 = null, + IStreamArgumentParser? parser4 = null + ) { + var meta = new DataStreamMetadata() { Name = name }; + parser1 ??= StreamArgumentParser.Default(); + parser2 ??= StreamArgumentParser.Default(); + parser3 ??= StreamArgumentParser.Default(); + parser4 ??= StreamArgumentParser.Default(); + var sf = new SingleDataStreamFactory(meta, FactoryWrapper); + streamFactory.Add(sf); + + ViewStream? FactoryWrapper(IEnumerable args) { + if (!args.TryGetExactCount(out var a1, out var a2, out var a3, out var a4)) { + return null; + } + if (!parser1.CanAndTryParse(a2, out var v1, out var err1)) { + return null; + } + if (!parser2.CanAndTryParse(a2, out var v2, out var err2)) { + return null; + } + if (!parser3.CanAndTryParse(a3, out var v3, out var err3)) { + return null; + } + if (!parser4.CanAndTryParse(a4, out var v4, out var err4)) { + return null; + } + return factory(v1, v2, v3, v4); + } + } + + public static void Add(this DataStreamFactory streamFactory, string name, Func factory, IStreamArgumentParser parser1, IStreamArgumentParser parser2, IStreamArgumentParser parser3, IStreamArgumentParser parser4, IStreamArgumentParser parser5) { + var meta = new DataStreamMetadata() { Name = name }; + var sf = new SingleDataStreamFactory(meta, FactoryWrapper); + streamFactory.Add(sf); + + ViewStream? FactoryWrapper(IEnumerable args) { + if (!args.TryGetExactCount(out var a1, out var a2, out var a3, out var a4, out var a5)) { + return null; + } + if (!parser1.CanAndTryParse(a2, out var v1, out var err1)) { + return null; + } + if (!parser2.CanAndTryParse(a2, out var v2, out var err2)) { + return null; + } + if (!parser3.CanAndTryParse(a3, out var v3, out var err3)) { + return null; + } + if (!parser4.CanAndTryParse(a4, out var v4, out var err4)) { + return null; + } + if (!parser5.CanAndTryParse(a5, out var v5, out var err5)) { + return null; + } + return factory(v1, v2, v3, v4, v5); + } + } + + public static void Add(this DataStreamFactory streamFactory, string name, Func singleStreamfactory, ILogger? logger = null, StreamArgumentsParser.TryParse? tryParse = null) { + var meta = new DataStreamMetadata(name); + var parser = tryParse is null + ? new StreamArgumentsParser() + : StreamArgumentsParser.FromDelegate(tryParse); + var sf = new SingleDataStreamFactory(meta, singleStreamfactory, parser, logger); + streamFactory.Add(sf); + } + + public static ViewStream? Create(this IDataStreamFactory streamFactory, string streamId, params object?[] args) => + streamFactory.Create(streamId, args.Select((a, i) => new StreamArgument(null, i, a))); +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamMetadata.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamMetadata.cs new file mode 100644 index 000000000..41905667d --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamMetadata.cs @@ -0,0 +1,45 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using System; +using System.Reflection; + +public interface IDataStreamMetadata { + public string Name { get; } +} + +public class DataStreamMetadata : IDataStreamMetadata { + public DataStreamMetadata() { } + + public DataStreamMetadata(string name) => + Name = name; + + public string? Name { get; set; } + + string IDataStreamMetadata.Name => Name ?? string.Empty; + + public static DataStreamMetadata FromType(Type type) { + var friendlyName = type.GetCustomAttribute()?.friendlyName; + var rv = new DataStreamMetadata() { + Name = friendlyName ?? type.Name, + }; + return rv; + } +} + +public static class DataStreamMetadataExtensions { + public static bool IsMatch(this IDataStreamMetadata dsm, string name, bool ignoreCase = true, bool ignoreUnderscores = true) { + var strComp = ignoreCase ? StringComparison.CurrentCultureIgnoreCase : StringComparison.CurrentCulture; + var nameWithoutUnderscores = new Lazy(() => RemoveUnderscores(name)); + return IsNameMatch(dsm.Name); + + static string RemoveUnderscores(string s) => + s.Replace("_", ""); + + bool IsNameMatch(string otherName) => + string.Equals(name, otherName, strComp) + || (ignoreUnderscores && string.Equals(RemoveUnderscores(name), RemoveUnderscores(otherName), strComp)); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/IDataStreamFactory.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/IDataStreamFactory.cs new file mode 100644 index 000000000..28a8db0bb --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/IDataStreamFactory.cs @@ -0,0 +1,25 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using System.Collections.Generic; +using System.Linq; + +public interface IDataStreamFactory { + IEnumerable GetSupportedStreams(); + + ViewStream? Create(string streamId, IEnumerable args); + + bool CanCreate(string streamId) => + GetSupportedStreams().Any(s => s.IsMatch(streamId)); +} + +/// +/// This interface provides a way for IDataStreamFactory's that create ViewStreams +/// that depend specifically on the composite IDataStreamFactory. This was designed +/// specifically for the NLStreamDataFactory. +/// +public interface IRecursiveDataStreamFactory : IDataStreamFactory { + void OnCompositeFactory(IDataStreamFactory compositeFactory); +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/ISingleDataStreamFactory.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/ISingleDataStreamFactory.cs new file mode 100644 index 000000000..d2c277bbb --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/ISingleDataStreamFactory.cs @@ -0,0 +1,18 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using System.Collections.Generic; + +public interface ISingleStreamFactory : IDataStreamFactory { + IDataStreamMetadata Metadata { get; } + + ViewStream? Create(IEnumerable args); + + ViewStream? IDataStreamFactory.Create(string streamId, IEnumerable args) => + Metadata.IsMatch(streamId) ? Create(args) : null; + + IEnumerable IDataStreamFactory.GetSupportedStreams() => + new[] { Metadata }; +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/IStreamArgumentParser.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/IStreamArgumentParser.cs new file mode 100644 index 000000000..957baf61f --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/IStreamArgumentParser.cs @@ -0,0 +1,42 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using System; +using System.Diagnostics.CodeAnalysis; + +public interface IStreamArgumentParser { + bool CanParse(StreamArgument arg, Type outputType); + + bool TryParse(StreamArgument arg, Type outputType, out object? result, out string? error); +} + +public interface IStreamArgumentParser : IStreamArgumentParser { + bool IStreamArgumentParser.CanParse(StreamArgument arg, Type outputType) => + outputType == typeof(T) || outputType.IsAssignableFrom(typeof(T)); + + bool CanParse(StreamArgument arg); + + bool TryParse(StreamArgument arg, [MaybeNullWhen(false)]out T result, out string? error); + + bool IStreamArgumentParser.TryParse(StreamArgument arg, Type outputType, out object? result, out string? error) { + if (outputType != typeof(T) || !outputType.IsAssignableFrom(typeof(T))) { + result = null; + error = $"OutputType '{outputType}' is assignable from type '{typeof(T)}'"; + return false; + } + bool rv = TryParse(arg, out T? resultT, out error); + result = resultT; + return rv; + } +} + +static class StreamArgumentParserExtensions { + public static bool CanAndTryParse(this IStreamArgumentParser parser, StreamArgument arg, [MaybeNullWhen(false)]out T result, out string? error) { + if (!parser.CanParse(arg)) { + result = default; + error = null; + return false; + } + return parser.TryParse(arg, out result, out error); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/IStreamArgumentsParser.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/IStreamArgumentsParser.cs new file mode 100644 index 000000000..a26821def --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/IStreamArgumentsParser.cs @@ -0,0 +1,19 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +public interface IStreamArgumentsParser { + bool TryParse(IReadOnlyList args, [NotNullWhen(true)]out object? result, out string? error); +} + +public interface IStreamArgumentsParser : IStreamArgumentsParser { + bool TryParse(IReadOnlyList args, [NotNullWhen(true)]out T? result, out string? error); + + bool IStreamArgumentsParser.TryParse(IReadOnlyList args, [NotNullWhen(true)]out object? result, out string? error) { + bool rv = TryParse(args, out T? resultT, out error); + result = resultT; + return rv; + } +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs index a889290cb..ab0a03f9a 100644 --- a/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs @@ -8,33 +8,36 @@ namespace BCG { using System.Collections; using System.Reflection; using System.IO; + using System.Collections.Generic; + using System.Linq; + using bts.collections; + using System.Diagnostics.CodeAnalysis; + using Spark.Core.Collections; using Bts.DataStreamCore; + using Microsoft.Extensions.Options; - public class PrimitiveManager { - static MarketDataStreamFactory mdStreamFactory; - private IDictionary myTypeLookupTable = new Hashtable(); + public class PrimitiveManager : IDataStreamFactory { private static PrimitiveManager mySelf; - private ArrayList myAssemblyNames = new ArrayList(); + readonly List myAssemblyNames = new(); + readonly Dictionary myTypeLookupTable = new(); - private PrimitiveManager() { - myLoadAllTypes(); - } + public PrimitiveManager(IOptions config) : this(config.Value) { } - private void myLoadAllTypes() { + public PrimitiveManager(PrimitiveManagerConfig config) { AppDomain currentDomain = AppDomain.CurrentDomain; currentDomain.AssemblyResolve += new ResolveEventHandler(myAssemblyResolver); loadAssembly(this.GetType().Assembly.FullName); - var assemblies = new string[] { "DataStream.dll", "SharedStreams.dll" }; - var config_assemblies = Configuration.get().Assemblies; - bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes config_assemblies: {config_assemblies}"); - foreach (string assemblyName in assemblies/*config_assemblies*/) { + foreach (string assemblyName in config.Assemblies.empty_if_null()) { bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes assemblyName: {assemblyName}"); string resolvedAssemblyPath = assemblyName; if (!System.IO.Path.IsPathRooted(assemblyName)) { resolvedAssemblyPath = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assemblyName)); } - loadAssembly(resolvedAssemblyPath); + loadAssembly(resolvedAssemblyPath, IncludeType); } + + bool IncludeType(Type t) => + !config.ExcludeTypes.empty_if_null().Any(tname => tname == t.Name); } private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) { @@ -48,12 +51,7 @@ namespace BCG { /// Loads all the Stream types associated with the assembly /// /// - public void loadAssembly(string assemblyName) { - Func is_blue_orb = (t) => { - //if (!assemblyName.Contains("SharedStreams.dll")) - // return true; - return t.Namespace == "bts.BlueOrb";// || t.Namespace == "BCG"; - }; + public void loadAssembly(string assemblyName, Func typeFilter = null) { bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}"); if (myAssemblyNames.Contains(assemblyName)) return; @@ -62,8 +60,12 @@ namespace BCG { GetType().Assembly : Assembly.LoadFrom(assemblyName); - foreach (Type type in loadedAssembly.GetTypes()) - if (!type.IsInterface && is_blue_orb(type) + IEnumerable types = loadedAssembly.GetTypes(); + if (typeFilter is not null) { + types = types.Where(typeFilter); + } + foreach (Type type in types) + if (!type.IsInterface && !type.IsAbstract && type.IsSubclassOf(typeof(ViewStream)) && type.IsPublic) { @@ -79,7 +81,7 @@ namespace BCG { break; } if (isVisible) { - // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName); + Log.assert_d(!myTypeLookupTable.ContainsKey(userRep)); myTypeLookupTable[userRep] = type; } } @@ -95,26 +97,25 @@ namespace BCG { return retVal; } + public IEnumerable GetSupportedTypes() => + myTypeLookupTable.Values.Cast(); + public IEnumerable GetSupportedStreams() => + GenericDictionaryWrapper.Create(myTypeLookupTable).Select(kv => new DataStreamMetadata(kv.Key)); - public static PrimitiveManager instance { - get { return mySelf = mySelf == null ? new PrimitiveManager() : mySelf; } - } + public Type ResolveType(string streamId) => + TryResolveType(streamId, out var t) ? t + : throw new ArgumentException($"Failed to resolve ViewStream type '{streamId}'", nameof(streamId)); - public static void SetMarketDataClient(object value) { - Log.isnull(mdStreamFactory); - mdStreamFactory = new(value); - } - - /// - /// Returns the Type name corresponding to the primitive name - /// - /// - /// - public string find(string friendlyName) { - if (myTypeLookupTable[friendlyName] == null) - throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types"); - return (myTypeLookupTable[friendlyName] as Type).Name; + public bool TryResolveType(string streamId, [NotNullWhen(true)] out Type result) { + result = GetSupportedStreams() + .Where(s => s.IsMatch(streamId)) + .Select(s => s.Name) + .Prepend(streamId) + .Select(myTypeLookupTable.GetValueOrDefault) + .ExceptNull() + .FirstOrDefault(); + return result is not null; } /// @@ -130,19 +131,110 @@ namespace BCG { constructorArgs = new object[args.Count]; args.CopyTo(constructorArgs, 0); } - var type = myTypeLookupTable[friendlyName] as Type - ?? throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types"); - if (type.HasCustomAttribute()) { - if (mdStreamFactory is null) { - Log.Gen.error("Live md not available for streams"); - return null; - } - return mdStreamFactory.Create(type, constructorArgs); + if (myTypeLookupTable[friendlyName] == null) + throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types"); + try { + return + constructorArgs == null ? + Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type) as ViewStream : + Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type, constructorArgs) as ViewStream; + } catch (Exception ex) { + Log.Gen.warn($"PrimitiveManager failed to construct ViewStream type '{friendlyName}': {ex}"); + } + return null; + } + + public ViewStream Create(string streamId, IEnumerable args) => + TryResolveType(streamId, out var type) ? Create(type, args) : null; + + /// + /// Constructs and returns a ViewStream of type corresponding to the friendly name + /// + /// friendly primitive name + /// args can be null or an empty ICollection if the paramterless constructor + /// is the desired one + /// + public ViewStream Create(Type type, IEnumerable args) { + var ctrs = type.GetConstructors(BindingFlags.Instance | BindingFlags.Public); + var argList = args.ToList(); + const int MAX_ALLOWED_ARGS = 64; + if (argList.Count > MAX_ALLOWED_ARGS) { + throw new ArgumentException( + $"ViewStream arg count ({argList.Count}) exceeds max allowed ({MAX_ALLOWED_ARGS})", nameof(args)); + } + if (argList.Any(a => !(a.Name?.Length > 0) && !a.Index.HasValue)) { + throw new ArgumentException( + "One or more ViewStream args lack both Name and Index (unable to assign a constructor param)", nameof(args)); + } + var indexedArgs = argList.Where(a => a.Index.HasValue).ToList(); + var namedArgs = argList.Where(a => a.Name?.Length > 0).ToList(); + if (!namedArgs.Select(a => a.Name).IsDistinct()) { + throw new ArgumentException("ViewStream arg names are not unique"); + } + if (!indexedArgs.Select(a => a.Index.Value).IsDistinct()) { + throw new ArgumentException("ViewStream arg indices are not unique"); + } + // Constructor params with default values may be omitted from the input 'args'. Therefore, + // 'minArgCount' >= 'argList.Count'. + var minArgCount = Math.Max(indexedArgs.Select(a => a.Index.Value).MaxOrDefault(), argList.Count); + foreach (var ctr in ctrs) { + if (TryAssignConstructorArgs(ctr, minArgCount, argList, indexedArgs, namedArgs, out var ctrArgs)) { + return ctr.Invoke(ctrArgs) as ViewStream; + } + } + return null; + + static bool TryAssignConstructorArgs( + ConstructorInfo ctr, + int minArgCount, + List argList, + List indexedArgs, + List namedArgs, + [NotNullWhen(true)] out object[] ctrArgs + ) { + ctrArgs = null; + var ctrParams = ctr.GetParameters(); + if (ctrParams.Length < minArgCount) { + return false; + } + var ctrArgList = new List(); + int remInputArgs = argList.Count; + for (int i = 0; i < ctrParams.Length; i++) { + var ctrParam = ctrParams[i]; + var paramType = ctrParams[i].ParameterType; + if (indexedArgs.TryGetFirst(a => CanAssignInputArg(ctrParam, a), out var a)) { + remInputArgs--; + ctrArgList.Add(a.Value); + } else if (ctrParam.HasDefaultValue) { + ctrArgList.Add(ctrParam.DefaultValue); + } else { + return false; + } + } + if (remInputArgs > 0) { + return false; + } + Log.assert(ctrArgList.Count == ctrParams.Length); + ctrArgs = ctrArgList.ToArray(); + return true; + } + + static bool CanAssignInputArg(ParameterInfo pi, StreamArgument arg) { + if (!string.IsNullOrEmpty(arg.Name) && arg.Name != pi.Name) { + return false; + } + if (arg.Index.HasValue && arg.Index.Value != pi.Position) { + return false; + } + if (arg.Value is null) { + if (!pi.ParameterType.IsValueType && !pi.ParameterType.IsNullable()) { + return false; + } + } else if (!pi.ParameterType.IsAssignableFrom(arg.Value.GetType())) { + return false; + } + return true; } - return - constructorArgs == null ? - Activator.CreateInstance(type) as ViewStream : - Activator.CreateInstance(type, constructorArgs) as ViewStream; } /// diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManagerConfig.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManagerConfig.cs new file mode 100644 index 000000000..881433190 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManagerConfig.cs @@ -0,0 +1,9 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using System.Collections.Generic; + +public class PrimitiveManagerConfig { + public List? Assemblies { get; set; } + public List? ExcludeTypes { get; set; } +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/SingleDataStreamFactory.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/SingleDataStreamFactory.cs new file mode 100644 index 000000000..3b2ad87e8 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/SingleDataStreamFactory.cs @@ -0,0 +1,70 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using Microsoft.Extensions.Logging; + +using System; +using System.Collections.Generic; + +class SingleDataStreamFactory : ISingleStreamFactory { + readonly Func, ViewStream?> factory; + + public SingleDataStreamFactory(IDataStreamMetadata metadata, Func, ViewStream?> factory) { + Metadata = metadata; + this.factory = factory; + } + + public IDataStreamMetadata Metadata { get; } + + public static SingleDataStreamFactory FromDelegate(IDataStreamMetadata metadata, Func factory) { + return new SingleDataStreamFactory(metadata, FactoryWrapper); + + ViewStream? FactoryWrapper(IEnumerable _) => + factory(); + } + + public ViewStream? Create(IEnumerable args) => + factory(args); + + public IEnumerable GetSupportedStreams() => + new[] { Metadata }; +} + +class SingleDataStreamFactory : ISingleStreamFactory { + readonly Func factory; + readonly IStreamArgumentsParser parser; + readonly ILogger? logger; + + public SingleDataStreamFactory(IDataStreamMetadata metadata, Func factory) + : this(metadata, factory, null) { } + + public SingleDataStreamFactory(IDataStreamMetadata metadata, Func factory, ILogger? logger) + : this(metadata, factory, new StreamArgumentsParser(), logger) { } + + public SingleDataStreamFactory(IDataStreamMetadata metadata, Func factory, IStreamArgumentsParser parser, ILogger? logger) { + Metadata = metadata; + this.factory = factory; + this.parser = parser; + this.logger = logger; + } + + public IDataStreamMetadata Metadata { get; } + + public ViewStream? Create(IEnumerable args) { + var argList = args as IReadOnlyList ?? args.ToReadOnlyList(); + if (!parser.TryParse(argList, out TParams? streamParams, out var err)) { + logger?.LogInvalidParams(Metadata.Name, err); + return null; + } + return Create(streamParams); + } + + public ViewStream? Create(TParams streamParams) { + return factory(streamParams); + } + + public IEnumerable GetSupportedStreams() => + new[] { Metadata }; +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgument.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgument.cs new file mode 100644 index 000000000..b923048d6 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgument.cs @@ -0,0 +1,18 @@ +#nullable enable +namespace Bts.DataStreamCore; + +public record struct StreamArgument(string? Name, int? Index, object? Value) { + public static implicit operator (string? Name, int? Index, object? Value)(StreamArgument arg) => + (arg.Name, arg.Index, arg.Value); + + public static implicit operator StreamArgument((string? Name, int? Index, object? Value) arg) => + new(arg.Name, arg.Index, arg.Value); + + public static StreamArgument Empty => new(null, null, null); + + public void Deconstruct(out string? name, out int? index, out object? value) { + name = Name; + index = Index; + value = Value; + } +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgumentAttribute.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgumentAttribute.cs new file mode 100644 index 000000000..79dffb0e6 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgumentAttribute.cs @@ -0,0 +1,29 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using System; +using System.Runtime.CompilerServices; + +[AttributeUsage(AttributeTargets.Property | AttributeTargets.Parameter)] +public class StreamArgumentAttribute : Attribute { + object? defaultValue; + bool hasDefaultValue; + + public StreamArgumentAttribute([CallerMemberName]string name = "") => + Name = name; + + public string Name { get; } + public int Position { get; set; } = -1; + public bool IsRequired { get; set; } + public bool AllowCommaSeparatedAliases { get; set; } = true; + public Type? ParserType { get; set; } + public object? DefaultValue { + get => defaultValue; + set { + defaultValue = value; + hasDefaultValue = true; + } + } + public bool HasDefaultValue => hasDefaultValue; +} + diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgumentConstructorAttribute.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgumentConstructorAttribute.cs new file mode 100644 index 000000000..4688fb065 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgumentConstructorAttribute.cs @@ -0,0 +1,15 @@ +namespace Bts.DataStreamCore; + +using System; + +[AttributeUsage(AttributeTargets.Constructor)] +public class StreamArgumentConstructorAttribute : Attribute { + public StreamArgumentConstructorAttribute() + : this(true) { } + + public StreamArgumentConstructorAttribute(bool isVisible) { + IsVisible = isVisible; + } + + public bool IsVisible { get; } +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgumentParser.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgumentParser.cs new file mode 100644 index 000000000..5f070aeea --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgumentParser.cs @@ -0,0 +1,63 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using System; +using System.Diagnostics.CodeAnalysis; + +public static class StreamArgumentParser { + public delegate bool TryParse(StreamArgument arg, [MaybeNullWhen(false)] out T result, [NotNullWhen(false)] out string? errorMessage); + + public static StreamArgumentParser Default() => + new(defaultCanParse, defaultTryParse); + + public static StreamArgumentParser Create(TryParse? tryParse = null) => + new(defaultCanParse, tryParse ?? defaultTryParse); + + public static StreamArgumentParser Create(Func? canParse, TryParse? tryParse = null) => + new(canParse ?? defaultCanParse, tryParse ?? defaultTryParse); + + static bool defaultCanParse(StreamArgument arg) { + var val = arg.Value; + if (val is null) { + return !typeof(T).IsValueType || (typeof(T).IsGenericType && typeof(T).GetGenericTypeDefinition() == typeof(Nullable<>)); + } + return val.GetType() == typeof(T); + } + + static bool defaultTryParse(StreamArgument arg, [MaybeNullWhen(false)] out T result, [NotNullWhen(false)] out string? error) { + var val = arg.Value; + if (val is null) { + if (!typeof(T).IsValueType || (typeof(T).IsGenericType && typeof(T).GetGenericTypeDefinition() == typeof(Nullable<>))) { + result = (T?)val!; + error = null; + return true; + } + } else { + try { + result = (T)val; + error = null; + return true; + } catch (InvalidCastException) { } + } + var valTypeName = val?.GetType()?.ToString() ?? "null"; + error = $"Unable to convert arg '{arg.Name}' of type {valTypeName} to type {typeof(T)}"; + result = default; + return false; + } +} + +public class StreamArgumentParser : IStreamArgumentParser { + readonly Func canParse; + readonly StreamArgumentParser.TryParse tryParse; + + public StreamArgumentParser(Func canParse, StreamArgumentParser.TryParse tryParse) { + this.canParse = canParse; + this.tryParse = tryParse; + } + + public bool CanParse(StreamArgument arg) => + canParse(arg); + + public bool TryParse(StreamArgument arg, [MaybeNullWhen(false)] out T result, [NotNullWhen(false)] out string? error) => + tryParse(arg, out result, out error); +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgumentsParser.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgumentsParser.cs new file mode 100644 index 000000000..139c02b2c --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgumentsParser.cs @@ -0,0 +1,198 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using bts; +using bts.collections; + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Reflection; + +public static class StreamArgumentsParser { + public delegate bool TryParse(IEnumerable args, [NotNullWhen(true)] out T? result, [NotNullWhen(false)] out string? errorMessage); + + public static IStreamArgumentsParser FromValue(T value) where T : notnull { + return FromDelegate(TryParse); + + bool TryParse(IEnumerable args, [NotNullWhen(true)] out T result, [NotNullWhen(false)] out string? errorMessage) { + result = value; + errorMessage = null; + return true; + } + } + + public static IStreamArgumentsParser FromDelegate(TryParse tryParse) => + new DelegateImpl(tryParse); + + class DelegateImpl : IStreamArgumentsParser { + readonly TryParse tryParse; + + public DelegateImpl(TryParse tryParse) => + this.tryParse = tryParse; + + public bool TryParse(IReadOnlyList args, [NotNullWhen(true)] out T? result, out string? error) => + tryParse(args, out result, out error); + } +} + +public class StreamArgumentsParser : IStreamArgumentsParser { + public bool TryParse(IReadOnlyList args, [NotNullWhen(true)] out T? result, out string? error) { + // indexedArgs has every arg with a non-null index. + // namedArgs has every arg with a non-null/empty name (except those with non-null indices). + // unnamedArgs have the others. + // Use the index as an id to prevent using the same arg twice. All named and + // indexed params must bind to a specific param in a constructor (the one with + // the matching name and/or index). Unnamed/indexed params are then bound to the + // remaining params (in the same order relative to their enumeration). + var allArgs = args.ToImmutableList(); + if (!allArgs.Select(a => a.Name).WhereNot(string.IsNullOrEmpty).IsDistinct()) { + throw new ArgumentException("Arg names are not unique"); + } + if (!allArgs.Select(a => a.Index).SelectWhereValues().IsDistinct()) { + throw new ArgumentException("Arg indices are not unique", nameof(args)); + } + int nArgs = args.Count; + error = null; + // Constructor params with default values may be omitted from the input 'args'. Therefore, + // 'minArgCount' >= 'argList.Count'. + var minArgCount = Math.Max(allArgs.Select(a => a.Index).SelectWhereValues().MaxOrDefault(), nArgs); + var streamType = typeof(T); + var ctrs = streamType.GetConstructors(); + foreach (var ctr in ctrs) { + var ctrAtt = ctr.GetCustomAttribute(); + if (ctrAtt is not null && !ctrAtt.IsVisible) { + continue; + } + var ps = ctr.GetParameters(); + if (ps.Length < minArgCount) { + continue; + } + if (TryAssignConstructorArgs(ps, allArgs, out var ctrArgs, out var unusedArgs)) { + if (unusedArgs.Any(a => string.IsNullOrEmpty(a.Name))) { + continue; + } + var resObj = ctr.Invoke(ctrArgs); + result = (T)resObj; + var resProps = typeof(T).GetProperties() + .Select(p => (Prop: p, Att: p.GetCustomAttribute())) + .Where(p => p.Att is not null) + .Select(p => (p.Prop, Att: p.Att!)) + .ToImmutableList(); + foreach (var arg in unusedArgs) { + // var argName = arg.Name; + // if (string.IsNullOrEmpty(argName)) { + // continue; + // } + var argName = arg.Name!; + if (resProps.TryGetFirst(pTup => IsNameMatch(argName, pTup), out var pTup, out var pIndex)) { + resProps = resProps.RemoveAt(pIndex); + } else { + return false; + } + } + return true; + + bool IsNameMatch(string argName, (PropertyInfo Prop, StreamArgumentAttribute Att) propTup) { + var (pi, att) = propTup; + if (att is null) { + return false; + } + var name = att.Name; + if (StringComparer.CurrentCultureIgnoreCase.Equals(name, argName)) { + return true; + } + return false; + } + } + } + result = default; + return false; + + static bool TryAssignConstructorArgs( + ParameterInfo[] ctrParams, + ImmutableList args, + [NotNullWhen(true)] out object?[]? ctrArgs, + out ImmutableList unusedArgs + ) { + ctrArgs = null; + var ctrArgList = new List(); + int nArgs = args.Count; + // Use locUsedArgs var inside lambdas (can't use ref params) + var locUsedArgs = new HashSet(); + for (int i = 0; i < ctrParams.Length; i++) { + var ctrParam = ctrParams[i]; + var paramType = ctrParams[i].ParameterType; + var att = ctrParam.GetCustomAttribute(); + var parserType = att?.ParserType; + var parser = parserType is null + ? null + : Activator.CreateInstance(parserType) as IStreamArgumentParser; + // Try to match indexed args first, then named args, then fallback to (all) args. + // indexedArgs.TryRemoveFirst() + int argIdx = -1; + int idx = 0; + foreach (var a in args) { + if (!CanAssignInputArg(ctrParam, a, parser)) { + continue; + } + if (!string.IsNullOrEmpty(a.Name) || a.Index.HasValue) { + // We're done. Names/indices are unambiguous. This arg MUST bind to this param. + argIdx = idx; + break; + } + if (argIdx == -1) { + // If we don't find a matching named/indexed param, we'll use this one. + argIdx = idx; + } + idx++; + } + if (argIdx != -1) { + var arg = args[argIdx]; + args = args.RemoveAt(argIdx); + var val = arg.Value; + if (parser is not null) { + if (!parser.TryParse(arg, ctrParam.ParameterType, out var parsedVal, out var err)) { + throw new InvalidOperationException($"Failed to parse param '{ctrParam.Name}'"); + } + val = parsedVal; + } + ctrArgList.Add(val); + } else if (att is not null && att.HasDefaultValue) { + ctrArgList.Add(att.DefaultValue); + } else if (ctrParam.HasDefaultValue) { + ctrArgList.Add(ctrParam.DefaultValue); + } else { + unusedArgs = args; + return false; + } + } + Log.assert(ctrArgList.Count == ctrParams.Length); + ctrArgs = ctrArgList.ToArray(); + unusedArgs = args; + return true; + } + + static bool CanAssignInputArg(ParameterInfo pi, StreamArgument arg, IStreamArgumentParser? parser = null) { + if (!string.IsNullOrEmpty(arg.Name) && arg.Name != pi.Name) { + return false; + } + if (arg.Index.HasValue && arg.Index.Value != pi.Position) { + return false; + } + if (parser is not null) { + return parser.CanParse(arg, pi.ParameterType); + } + if (arg.Value is null) { + if (!pi.ParameterType.IsValueType && !pi.ParameterType.IsNullable()) { + return false; + } + } else if (!pi.ParameterType.IsAssignableFrom(arg.Value.GetType())) { + return false; + } + return true; + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamFactoryArgStore.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamFactoryArgStore.cs new file mode 100644 index 000000000..db4908cd3 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamFactoryArgStore.cs @@ -0,0 +1,282 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using mbts.utils; + +using Microsoft.Extensions.Logging; + +using Spark.Core; + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Text.RegularExpressions; + +public class StreamFactoryArgStore { + readonly ILogger? logger; + readonly string streamType; + ImmutableList args; + + public StreamFactoryArgStore(IEnumerable args, string? streamType = null, ILogger? logger = null) { + this.args = args.ToImmutableList(); + this.streamType = streamType ?? "anonStream"; + this.logger = logger; + } + + public delegate bool TryGetParamsDelegate(IEnumerable args, [NotNullWhen(true)] out T? result, [NotNullWhen(false)] out string? error); + public delegate bool TryConvert(object? obj, [NotNullWhen(true)] out T? result); + + public static bool TryGetParams(IEnumerable args, [NotNullWhen(true)] out T? result, [NotNullWhen(false)] out string? error) => + new StreamFactoryArgStore(args).TryGetParams(out result, out error); + + public bool TryGetParams([MaybeNull][NotNullWhen(true)] out T result, [NotNullWhen(false)] out string? errorMessage) { + var props = typeof(T).GetProperties(); + var res = Activator.CreateInstance(); + if (res is null) { + errorMessage = $"Failed to instantiate {typeof(T)}"; + result = default; + return false; + } + foreach (var p in props) { + var att = p.GetCustomAttribute(); + string name = att?.Name ?? p.Name; + if (TryGet(name, out var arg)) { + SetValue(arg); + } else if (att is not null && att.Position >= 0 && args.Skip(att.Position).TryGetFirst(out arg)) { + SetValue(arg!); + } else if (att?.IsRequired ?? false) { + errorMessage = $"Missing required param '{p.Name}'"; + result = default; + return false; + } + + void SetValue(StreamArgument arg) { + var (name, index, val) = arg; + var pType = p.PropertyType; + if (val is string valStr && att?.ParserType is Type parserType) { + var parser = ExceptionAssert.of_type(Activator.CreateInstance(parserType)); + if (!parser.TryParse(arg, pType, out val, out var error)) { + throw new InvalidOperationException($"Failed to parse arg with custom parser of type {parserType}. {error}"); + } + } + bool isNullable = pType.IsNullable(); + if (isNullable) { + pType = pType.GetGenericArguments().Single(); + } + var vType = val?.GetType() ?? typeof(object); + try { + if (vType.IsAssignableTo(pType) || (val is null && (!pType.IsValueType || isNullable))) { + p.SetValue(res, val); + } else if (pType == typeof(DateTime)) { + SetValueType(BlueDataUtility.GetDateTime(val)); + } else if (pType == typeof(TimeSpan)) { + SetValueType(BlueDataUtility.GetTimeSpan(val)); + } else if (pType == typeof(string)) { + p.SetValue(res, val?.ToString()); + } else if (pType.IsEnum) { + p.SetValue(res, BlueDataUtility.GetEnum(pType, val)); + } else { + p.SetValue(res, Convert.ChangeType(val, pType)); + } + } catch (Exception ex) { + logger?.LogError("""[stream_factory_arg_parse_err.json]{{ "err": "{}", "ts": "{}" }}""", ex.ToString(), DateTime.Now); + } + + void SetValueType(TProp? pVal) where TProp : struct { + if (isNullable) { + p.SetValue(res, pVal); + } else { + p.SetValue(res, pVal.assert_value()); + } + } + } + } + errorMessage = null; + result = res; + return true; + } + + public TimeSpan? GetTimeSpan(string name) => + GetTimeSpan(true, new[] { name }); + + public TimeSpan? GetTimeSpan(bool required, string[] preferredNames) => + preferredNames + .Select(n => BlueDataUtility.GetTimeSpan(Get(n))) + .Where(ts => ts.HasValue) + .FirstOrDefault(); + + public IEnumerable GetStrings() => + args.Select(a => a.Value).OfType(); + + // First, parses the args associated with preferredNames and returns the first successful + // parse result. If none were successful, tries to parse every arg. + public T? GetEnum(params string[] preferredNames) where T : struct { + var candidateObjs = preferredNames.Select(n => Get(n)).Concat(args.Select(a => a.Value)); + foreach (var obj in candidateObjs) { + if (obj is T res) { + return res; + } + if (obj is string s && Enum.TryParse(s, true, out res)) { + return res; + } + } + return null; + } + + public IEnumerable GetDateTimes() => + args.Select(a => convertToDateTime(a.Value)).SelectWhereValues(); + + public bool GetDateTime(string name, [NotNullWhen(true)] out DateTime? result) { + result = GetDateTime(true, name); + return result.HasValue; + } + + public bool GetDateTime([NotNullWhen(true)] out DateTime? result, params string[] names) { + result = GetDateTime(names); + return result.HasValue; + } + + public bool GetDateTime([NotNullWhen(true)] out DateTime? result, out int index, bool removeArg) { + int i = 0; + foreach (var arg in args) { + var dt = convertToDateTime(arg.Value); + if (dt.HasValue) { + result = dt.Value; + index = i; + if (removeArg) { + args = args.RemoveAt(i); + } + return true; + } + } + result = null; + index = -1; + return false; + } + + public DateTime? GetDateTime(params string[] names) => + GetDateTime(true, names); + + public DateTime? GetDateTime(bool required, params string[] names) => + names.Select(n => Get(n)).Select(convertToDateTime).Where(dt => dt.HasValue).FirstOrDefault(); + + public object? Get(params string[] names) => + Get(true, names); + + public object? Get(out int index, params string[] names) => + Get(true, out index, names); + + public object? Get(bool required, params string[] names) => + Get(required, out _, names); + + public object? Get(bool required, out int index, params string[] names) { + if (names.Length == 0) { + index = -1; + return null; + } + if (!TryGet(names, out var result, out index) && required && logger is not null) { + logger.LogMissingParam(streamType, names[0]); + } + return result.Value; + } + + public bool TryGetRegex(string name, out StreamArgument result) => + TryGet(name, out result, checkDefaultMatch: false, checkRegexMatch: true); + + public bool TryGet(string name, out StreamArgument result, bool checkDefaultMatch = true, bool checkRegexMatch = false) => + TryGet(new[] { name }, out result, out _, checkDefaultMatch, checkRegexMatch); + + public bool TryGet(IEnumerable names, out StreamArgument result, out int index, bool checkDefaultMatch = true, bool checkRegexMatch = false) { + if (checkDefaultMatch) { + foreach (var name in names.SelectMany(Variants)) { + if (args.TryGetFirst(a => a.Name == name, out var a, out index)) { + result = a; + return true; + } + } + } + if (checkRegexMatch) { + foreach (var name in names) { + if (args.TryGetFirst(a => Regex.IsMatch(a.Name ?? string.Empty, name, RegexOptions.IgnoreCase), out var a, out index)) { + result = a; + return true; + } + } + } + index = -1; + result = default; + return false; + + IEnumerable Variants(string s) { + if (string.IsNullOrEmpty(s)) { + yield break; + } + yield return s; + // Convert to snake_case + var bd = new StringBuilder(char.ToLower(s[0])); + for (int i = 1; i < s.Length; i++) { + if (i > 1 && char.IsUpper(s[i]) && char.IsLower(s[i - 1])) { + bd.Append('_'); + } + bd.Append(char.ToLower(s[i])); + } + yield return bd.ToString(); + // Convert to camelCase + bd.Clear(); + for (int i = 0; i < s.Length; i++) { + int numUnderscores = 0; + while (s[i + numUnderscores] == '_' & (i + numUnderscores < s.Length)) { + numUnderscores++; + } + if (numUnderscores > 0) { + i += numUnderscores; + if (i >= s.Length) { + break; + } + bd.Append(char.ToUpper(s[i])); + } else { + bd.Append(s[i]); + } + } + yield return bd.ToString(); + } + } + + static DateTime? convertToDateTime(object? obj) => + tryConvert(obj, out DateTime dt) ? dt : null; + + static bool tryConvert(object? obj, out DateTime res) { + if (obj is DateTime dt) { + res = dt; + return true; + } + if (obj is string s) { + // if (OffsetDateTimePattern) + if (DateTimeOffset.TryParseExact(s, "yyyy-MM-ddTHH:mm:ssK", null, System.Globalization.DateTimeStyles.None, out var dto)) { + res = dto.ToUniversalTime().DateTime; + return true; + } + if (DateTimeOffset.TryParseExact(s, "yyyy-MM-ddTHH:mm:ss.fffffffffK", null, System.Globalization.DateTimeStyles.None, out dto)) { + res = dto.ToUniversalTime().DateTime; + return true; + } + if (DateTime.TryParse(s, null, System.Globalization.DateTimeStyles.RoundtripKind, out res)) { + return true; + } + if (DateTimeOffset.TryParse(s, out var resOff)) { + res = resOff.ToUniversalTime().DateTime; + return true; + } + return DateTime.TryParse(s, out res); + } + res = default; + return false; + } + +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamFactoryLogs.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamFactoryLogs.cs new file mode 100644 index 000000000..0b5e7047e --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamFactoryLogs.cs @@ -0,0 +1,25 @@ +namespace Bts.DataStreamCore; + +using Microsoft.Extensions.Logging; + +using System; + +public static partial class StreamFactoryLogs { + public static void LogCreatingStream(this ILogger logger, string streamType) => + logger.LogCreatingStream(streamType, DateTime.Now); + + public static void LogCreatingStreamError(this ILogger logger, string streamType, string error) => + logger.LogCreatingStreamError(streamType, error, DateTime.Now); + + [LoggerMessage(EventId = 1, Level = LogLevel.Warning, Message = """[crt_stream.json]{{ "typ": {StreamType}, "ts": {Timestamp} }}""")] + public static partial void LogCreatingStream(this ILogger logger, string streamType, DateTime timestamp); + + [LoggerMessage(EventId = 2, Level = LogLevel.Warning, Message = """[crt_stream_err.json]{{ "typ": {StreamType}, "err": {Error}, "ts": {Timestamp} }}""")] + public static partial void LogCreatingStreamError(this ILogger logger, string streamType, string error, DateTime timestamp); + + [LoggerMessage(EventId = 3, Level = LogLevel.Warning, Message = "Unable to create '{StreamType}' stream. Missing required parameter '{Parameter}'.")] + public static partial void LogMissingParam(this ILogger logger, string streamType, string parameter); + + [LoggerMessage(EventId = 4, Level = LogLevel.Warning, Message = "Unable to create '{StreamType}' stream. {error}.")] + public static partial void LogInvalidParams(this ILogger logger, string streamType, string error); +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/WellKnownStreamArgumentType.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/WellKnownStreamArgumentType.cs new file mode 100644 index 000000000..c9e586310 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/WellKnownStreamArgumentType.cs @@ -0,0 +1,58 @@ +#nullable enable +namespace Bts.DataStreamCore; + +public static class WellKnownStreamArgumentAliases { + /// + /// Aliases for a Venue argument. + /// + public const string Venue = "venue,exchange,exch"; + + /// + /// Aliases for a Type argument. + /// + public const string Asset = "symbol,sym,product,asset"; + + /// + /// Aliases for a Type argument. + /// + public const string Type = "type,typ"; + + /// + /// Aliases for a SecurityType argument. Some are potentially ambiguous, like 'type' and 'typ'. + /// If this conflicts with other arguments, consider using . + /// + public const string SecurityType = "secType,secTyp,securityType,securityTyp,type,typ"; + + /// + /// Aliases for a SecurityName argument. Some are potentially ambiguous, like 'name'. If this + /// conflicts with other arguments, consider using . + /// + public const string SecurityName = "name,sec,secName,security"; + + public const string QualifiedSecurityType = "secType,secTyp,securityType,securityTyp"; + public const string QualifiedSecurityName = "sec,secName,security"; + + public const string VenueRegex = "^(venue|exchange|exch)$"; + public const string AssetRegex = "^(symbol|sym|asset|product)$"; + public const string TypeRegex = "^(type|typ)$"; + public const string SecurityTypeRegex = "^(secType|secTyp|type|typ)$"; + public const string SecurityNameRegex = "^(sec|security|name|secName|securityName)$"; + + public const string ShowVenueRegex = "^(show|include)(Exch|Venue)$"; + public const string ShowAssetRegex = "^(show|include)(Symbol|Sym|Asset|Product)$"; + public const string ShowSecurityTypeRegex = "^(show|include)(Sec|Security)?(Type|Typ)$"; + public const string ShowSecurityNameRegex = "^(show|include)(Sec|Security|Name|SecName|SecurityName)$"; + public const string ShowExpirationRegex = "^(show|include)(Exp(iration|Date)?)$"; + public const string ShowCashPerPointRegex = "^(show|include)(CashPerPoint|Cpp)$"; + + public const string ShowLegVenueRegex = "^(show|include)Leg(Exch|Venue)$"; + public const string ShowLegSecurityTypeRegex = "^(show|include)Leg(Sec|Security)?(Type|Typ)$"; + public const string ShowLegAssetRegex = "^(show|include)Leg(Symbol|Sym|Asset|Product)$"; + public const string ShowLegExpirationRegex = "^(show|include)Leg(Exp(iration|Date)?)$"; + public const string ShowLegNameRegex = "^(show|include)Leg(Name|Desc)$"; + public const string ShowLegCashPerPointRegex = "^(show|include)Leg(CashPerPoint|Cpp)$"; + + public const string ReplaySpeedRegex = $"^(replay|replaySpeed|speed|rate|velocity)$"; + public const string BeginDateRegex = "(begin|start)(date(Time)?|time)?"; + public const string EndDateRegex = "(end|stop)(date(Time)?|time)?"; +} diff --git a/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs b/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs index fc036feb4..ead934004 100644 --- a/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs +++ b/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs @@ -3,6 +3,7 @@ namespace Bts.DataStreamCore.Services; using BCG; using BCG.Utilities.General; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; public static class DataStreamCoreServices { @@ -14,4 +15,11 @@ public static class DataStreamCoreServices { ViewStreamDatabase.SetGlobalInstance(new ViewStreamDatabase(sp.GetRequiredService())); return ViewStreamDatabase.instance; }); + + public static IServiceCollection AddPrimitiveStreams(this IServiceCollection services, IConfiguration configurationSection) => + services.Configure(configurationSection) + .AddSingleton(); + + public static IServiceCollection AddCompositeDataStreamFactory(this IServiceCollection services) => + services.AddSingleton(); } diff --git a/src/DataStreamSuite/DataStreamExpressions/Assignment.cs b/src/DataStreamSuite/DataStreamExpressions/Assignment.cs index 86590fad7..060b7ab6a 100644 --- a/src/DataStreamSuite/DataStreamExpressions/Assignment.cs +++ b/src/DataStreamSuite/DataStreamExpressions/Assignment.cs @@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions; using BCG; +using Bts.DataStreamCore; + class Assignment : StatementExpression { readonly StatementExpression expression; readonly string id; @@ -19,8 +21,8 @@ class Assignment : StatementExpression { return result; } - public override ViewStream? myGetView(IDictionary env) { - var result = expression.getView(env); + public override ViewStream? myGetView(IDictionary env, IDataStreamFactory streamFactory) { + var result = expression.getView(env, streamFactory); env[id] = expression; return result; } diff --git a/src/DataStreamSuite/DataStreamExpressions/DataStreamExpression.cs b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpression.cs index 7f84a8fad..842ff894d 100644 --- a/src/DataStreamSuite/DataStreamExpressions/DataStreamExpression.cs +++ b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpression.cs @@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions; using BCG; +using Bts.DataStreamCore; + using System.Text; public class DataStreamExpression { @@ -31,10 +33,10 @@ public class DataStreamExpression { return sb.ToString(); } - public ViewStream? getView(IDictionary env) { + public ViewStream? getView(IDictionary env, IDataStreamFactory streamFactory) { for (int i = 0; i < Statements.Count - 1; i++) { - Statements[i].getView(env); + Statements[i].getView(env, streamFactory); } - return Statements[^1].getView(env); + return Statements[^1].getView(env, streamFactory); } } diff --git a/src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs b/src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs index 1b90cd17e..c2cbc4429 100644 --- a/src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs +++ b/src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs @@ -2,7 +2,8 @@ namespace Bts.DataStreamExpressions; using BCG; -using System.Collections; +using Bts.DataStreamCore; + using System.Collections.Immutable; public class PrimitiveExpression : StatementExpression { @@ -20,7 +21,7 @@ public class PrimitiveExpression : StatementExpression { } public override string print(int startCol, IList activeBindings) { - string prelude = "new " + PrimitiveManager.instance.find(myName) + " ("; + string prelude = "new " + myName + " ("; return prelude + myPrintArgs(startCol + prelude.Length, activeBindings) + ")"; } @@ -31,21 +32,19 @@ public class PrimitiveExpression : StatementExpression { return prelude + myPrintViewArgs(startCol + prelude.Length) + ")"; } - public override ViewStream myGetView(IDictionary env) { - ArrayList resolvedArgs = new ArrayList(); + public override ViewStream? myGetView(IDictionary env, IDataStreamFactory streamFactory) { + var resolvedArgs = myArgs.Select((arg, i) => new StreamArgument(arg.Name, string.IsNullOrEmpty(arg.Name) ? i : null, ResolveArg(arg))); + return streamFactory.Create(myName, resolvedArgs); - foreach (var arg in myArgs) { + object? ResolveArg(Argument arg) { if (arg.StatementExp is not null) { - resolvedArgs.Add(arg.StatementExp.myGetView(env)); - continue; + return arg.StatementExp.myGetView(env, streamFactory); } if (arg.QuotedView is not null) { - resolvedArgs.Add(arg.QuotedView.Value); - continue; + return arg.QuotedView.Value; } - resolvedArgs.Add(arg.Obj); + return arg.Obj; } - return PrimitiveManager.instance.build(myName, resolvedArgs); } string myPrintSingleArg(int atColumn, object? o, IList activeBindings) { diff --git a/src/DataStreamSuite/DataStreamExpressions/StatementExpression.cs b/src/DataStreamSuite/DataStreamExpressions/StatementExpression.cs index 211c290bb..7eed8636c 100644 --- a/src/DataStreamSuite/DataStreamExpressions/StatementExpression.cs +++ b/src/DataStreamSuite/DataStreamExpressions/StatementExpression.cs @@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions; using BCG; +using Bts.DataStreamCore; + public abstract class StatementExpression { ViewStream? view; @@ -13,10 +15,10 @@ public abstract class StatementExpression { public virtual string? printView(int startCol) => null; - public ViewStream? getView(IDictionary env) => - view ??= myGetView(env); + public ViewStream? getView(IDictionary env, IDataStreamFactory streamFactory) => + view ??= myGetView(env, streamFactory); - public virtual ViewStream? myGetView(IDictionary env) => + public virtual ViewStream? myGetView(IDictionary env, IDataStreamFactory streamFactory) => null; public static string makeCSString(string s) { diff --git a/src/DataStreamSuite/DataStreamExpressions/ViewNameExpression.cs b/src/DataStreamSuite/DataStreamExpressions/ViewNameExpression.cs index 5d4e3f6e2..a1dea49d0 100644 --- a/src/DataStreamSuite/DataStreamExpressions/ViewNameExpression.cs +++ b/src/DataStreamSuite/DataStreamExpressions/ViewNameExpression.cs @@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions; using BCG; +using Bts.DataStreamCore; + public class ViewNameExpression : StatementExpression { readonly string id; @@ -14,6 +16,6 @@ public class ViewNameExpression : StatementExpression { public override string printView(int startCol) => id; - public override ViewStream? myGetView(IDictionary env) => - env.TryGetValue(id, out var exp) ? exp.getView(env) : new TableStream(id); + public override ViewStream? myGetView(IDictionary env, IDataStreamFactory streamFactory) => + env.TryGetValue(id, out var exp) ? exp.getView(env, streamFactory) : new TableStream(id); } diff --git a/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs index 7467cbd90..3ba431cd1 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs @@ -4,6 +4,7 @@ using BCG; using bts; +using Bts.DataStream; using Bts.DataStreamCore; using Bts.DataStreamExpressions; using Bts.PubSub; @@ -28,13 +29,28 @@ public sealed class DataStreamPublisherService : IHostedService, IDisposable { readonly DataStreamPublisherConfig config; readonly IPubTableBroker pubTableBroker; readonly IWorkScheduler workScheduler; + readonly IDataStreamFactory streamFactory; List? resources = new(); - public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, IOptions config) - : this(pubTableBroker, workScheduler, config.Value) { } + public DataStreamPublisherService( + IPubTableBroker pubTableBroker, + IEnumerable streamFactories, + IWorkScheduler workScheduler, + IOptions config + ) + : this(pubTableBroker, streamFactories, workScheduler, config.Value) { } - public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, DataStreamPublisherConfig config) { + public DataStreamPublisherService( + IPubTableBroker pubTableBroker, + IEnumerable streamFactories, + IWorkScheduler workScheduler, + DataStreamPublisherConfig config + ) { this.pubTableBroker = pubTableBroker; + var streamFactoryList = streamFactories.ToList(); + streamFactory = streamFactoryList.Count == 1 + ? streamFactoryList[0] + : new CompositeDataStreamFactory(streamFactoryList); this.workScheduler = workScheduler; this.config = config; } @@ -48,7 +64,7 @@ public sealed class DataStreamPublisherService : IHostedService, IDisposable { Log.Gen.warn("Skipping PublishedStream because it doesn't have a name (must define 'Name' in config json)."); continue; } - var vs = CreateViewStream(publishedStream); + var vs = CreateViewStream(streamFactory, publishedStream); if (vs is null) { continue; } @@ -72,7 +88,7 @@ public sealed class DataStreamPublisherService : IHostedService, IDisposable { resources?.Clear(); } - static ViewStream? CreateViewStream(PublishedStream ps) { + static ViewStream? CreateViewStream(IDataStreamFactory streamFactory, PublishedStream ps) { Log.Gen.info( $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}"); if (string.IsNullOrEmpty(ps.Name)) { @@ -80,7 +96,7 @@ public sealed class DataStreamPublisherService : IHostedService, IDisposable { $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}"); return null; } - var vs = buildStream(ps.Expression); + var vs = buildStream(streamFactory, ps.Expression); if (vs is null) { Log.Gen.warn( $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}"); @@ -89,14 +105,14 @@ public sealed class DataStreamPublisherService : IHostedService, IDisposable { return vs; } - static ViewStream? buildStream(string? streamExpression) { + static ViewStream? buildStream(IDataStreamFactory streamFactory, string? streamExpression) { if (streamExpression is null) { return null; } try { Log.Gen.info($"Building ViewStream: {streamExpression}"); var exp = DataStreamExpressionsInterpreter.Evaluate(streamExpression); - return exp.getView(new Dictionary(0)); + return exp.getView(new Dictionary(0), streamFactory); } catch (Exception ex) { Log.assert(false, $"Could not build stream. expression: {streamExpression}; ex: {ex}"); return null; diff --git a/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherServices.cs b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherServices.cs index 1c1e9d1a4..a65913936 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherServices.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherServices.cs @@ -1,24 +1,10 @@ namespace Bts.DataStreamPublisher.Services; -using Bts.DataStream.Services; -using Bts.DataStreamCore.Services; -using Bts.PubSub.Services; - using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Spark.Api.Services; -using Spark.Core.Services; - public static class DataStreamPublisherServices { public static IServiceCollection AddDataStreamPublisher(this IServiceCollection services, IConfiguration configurationSection) => - services.AddSpark(configurationSection.GetSection("Spark")) - .AddSparkApi(configurationSection.GetSection("SparkApi")) - .AddMasterWorker(configurationSection.GetSection("MasterWorker")) - .AddViewStreamDatabase() - .AddPubSubClient(configurationSection.GetSection("PubSub")) - .AddPubTableBroker() - .AddRemoteQueryManager() - .Configure(configurationSection.GetSection("DataStreamPublisher")) + services.Configure(configurationSection) .AddHostedService(); } diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs index e4215f5df..625826e96 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs @@ -22,8 +22,7 @@ using Bts.DataStreamExpressions; // using BCG.Evaluator.DataStreamExpression; // using BCG.Utilities.Win32; using System.Threading.Tasks; - - +using Bts.DataStreamCore; namespace BCG { @@ -49,9 +48,16 @@ namespace BCG private ViewStream? myAllTagsStream; private string myKQ; private TextWriter? myConsoleOutput; + readonly PrimitiveManager primitiveManager; + readonly IDataStreamFactory streamFactory; - public MainForm() + public MainForm(PrimitiveManager primitiveManager) + : this(primitiveManager, primitiveManager) { } + + public MainForm(PrimitiveManager primitiveManager, IDataStreamFactory streamFactory) { + this.primitiveManager = primitiveManager; + this.streamFactory = streamFactory; //Task.Run(async () => await _run()); InitializeComponent(); // myKQ = ConfigurationRequester.getApplicationQualifier(); @@ -109,11 +115,11 @@ namespace BCG private IDictionary myGetLibraryData() { SortedList retVal = new SortedList(); - string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames(); + string[] primitiveNames = primitiveManager.getPrimitiveNames(); foreach (string pName in primitiveNames) { - string aName = PrimitiveManager.instance.getAssemblyName(pName); - ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName); + string aName = primitiveManager.getAssemblyName(pName); + ICollection cNames = primitiveManager.getConstructorSigns(pName); if (!retVal.Contains(aName)) retVal.Add(aName, new SortedList()); (retVal[aName] as IDictionary)[pName] = new ArrayList(cNames); @@ -667,7 +673,7 @@ namespace BCG DialogResult dr = ofd.ShowDialog(this); if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK) { - PrimitiveManager.instance.loadAssembly(ofd.FileName); + primitiveManager.loadAssembly(ofd.FileName); myLoadAssemblies(); } } diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs index 777f181f4..d7f102613 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs @@ -4,6 +4,7 @@ using bts; using bts.api.core; using bts.md; +using Bts.DataStreamCore; using Bts.DataStreamQueryAnalyzer; using Bts.PubSub; @@ -29,7 +30,14 @@ static class Program { await Configuration.InitializeAsync(cfg.Sparkd); PrimitiveManager.SetMarketDataClient(new MarketDataClient(new(TimeSpan.FromSeconds(0.5)))); Log.Gen.info("Starting DataStreamQueryAnalyzer"); - var mainForm = new MainForm(); + var primitiveManager = new PrimitiveManager( + new PrimitiveManagerConfig() { + Assemblies = new() { + "DataStream.dll", + "SharedStreams.dll" + }, + }); + var mainForm = new MainForm(primitiveManager); if (cfg.StreamFile is not null) { mainForm.loadFile(cfg.StreamFile); } diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BboPriceSummaryRow.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BboPriceSummaryRow.cs new file mode 100644 index 000000000..e6344325b --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BboPriceSummaryRow.cs @@ -0,0 +1,341 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using bts.api; +using bts.co2; +using bts.utils; + +using Bts.DataStreamCore; +using Bts.SharedStreams; + +using Spark.Core.Collections; +using Spark.Core.Reactive; + +using System.Collections.Immutable; + +public class BboPriceSummaryRow : IBlueRow { + public static readonly IBlueTableSchema Schema = + new BlueTableSchemaBuilder("BboPriceSummary") { + (MyCols.IdName, typeof(int)), + WellKnownColumns.Timestamp, + (MyCols.BidTimeName, typeof(DateTime)), + (MyCols.BidTimeTilTradeName, typeof(TimeSpan)), + WellKnownColumns.BidPrice, + WellKnownColumns.BidQuantity, + (MyCols.BidMaxQuantityName, typeof(double)), + (MyCols.BidTradeQuantityName, typeof(double)), + (MyCols.BidInfluxQuantityName, typeof(double)), + (MyCols.BidUnfilledQuantityName, typeof(double)), + (MyCols.AskTimeName, typeof(DateTime)), + (MyCols.AskTimeTilTradeName, typeof(TimeSpan)), + WellKnownColumns.AskPrice, + WellKnownColumns.AskQuantity, + (MyCols.AskMaxQuantityName, typeof(double)), + (MyCols.AskTradeQuantityName, typeof(double)), + (MyCols.AskInfluxQuantityName, typeof(double)), + (MyCols.AskUnfilledQuantityName, typeof(double)), + (MyCols.TradeQuantityName, typeof(double)), + (MyCols.AvgTradePriceName, typeof(double)), + } + .ToSchema(); + + readonly ImmutableDictionary books; + readonly ImmutableSortedSet> bookTimeseries; + readonly ImmutableDictionary trades; + readonly ImmutableSortedSet> tradeTimeseries; + + public BboPriceSummaryRow(int id) + : this( + id, + ImmutableDictionary.Empty, + ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<(DateTime Timestamp, int Id), BookRowData>(TupleComparer.Comparer())), + ImmutableDictionary.Empty, + ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<(DateTime Timestamp, int Id), TradeRowData>(TupleComparer.Comparer()))) { } + + public BboPriceSummaryRow( + int id, + ImmutableDictionary books, + ImmutableSortedSet> bookTimeseries, + ImmutableDictionary trades, + ImmutableSortedSet> tradeTimeseries + ) { + Id = id; + this.books = books; + this.bookTimeseries = bookTimeseries; + this.trades = trades; + this.tradeTimeseries = tradeTimeseries; + if (bookTimeseries.TryGetFirst(out var bk)) { + BidPrice = bk.Value.BidPrice; + BidQuantity = bk.Value.BidQuantity; + BidMaxQuantity = bookTimeseries.Select(kv => kv.Value.BidQuantity).Except(double.NaN).MaxOrDefault(); + AskPrice = bk.Value.AskPrice; + AskQuantity = bk.Value.AskQuantity; + AskMaxQuantity = bookTimeseries.Select(kv => kv.Value.AskQuantity).Except(double.NaN).MaxOrDefault(); + } + } + + public object? this[int index] => + index switch { + MyCols.IdIndex => Id, + MyCols.TimestampIndex => OpenTimestamp, + MyCols.BidTimeIndex => BidTime, + MyCols.BidTimeTilTradeIndex => BidTimeTilTrade, + MyCols.BidPriceIndex => BidPrice, + MyCols.BidQuantityIndex => BidQuantity, + MyCols.BidMaxQuantityIndex => BidMaxQuantity, + MyCols.BidTradeQuantityIndex => BidTradeQuantity, + MyCols.BidInfluxQuantityIndex => BidInfluxQuantity, + MyCols.BidUnfilledQuantityIndex => BidUnfilledQuantity, + MyCols.AskTimeIndex => AskTime, + MyCols.AskTimeTilTradeIndex => AskTimeTilTrade, + MyCols.AskPriceIndex => AskPrice, + MyCols.AskQuantityIndex => AskQuantity, + MyCols.AskMaxQuantityIndex => AskMaxQuantity, + MyCols.AskTradeQuantityIndex => AskTradeQuantity, + MyCols.AskInfluxQuantityIndex => AskInfluxQuantity, + MyCols.AskUnfilledQuantityIndex => AskUnfilledQuantity, + MyCols.TradeQuantityIndex => TradeQuantity, + MyCols.AvgTradePriceIndex => AvgTradePrice, + _ => throw new KeyNotFoundException(), + }; + + public object? this[string colName] => + colName switch { + MyCols.IdName => Id, + MyCols.TimestampName => OpenTimestamp, + MyCols.BidTimeName => BidTime, + MyCols.BidTimeTilTradeName => BidTimeTilTrade, + MyCols.BidPriceName => BidPrice, + MyCols.BidQuantityName => BidQuantity, + MyCols.BidMaxQuantityName => BidMaxQuantity, + MyCols.BidTradeQuantityName => BidTradeQuantity, + MyCols.BidInfluxQuantityName => BidInfluxQuantity, + MyCols.BidUnfilledQuantityName => BidUnfilledQuantity, + MyCols.AskTimeName => AskTime, + MyCols.AskTimeTilTradeName => AskTimeTilTrade, + MyCols.AskPriceName => AskPrice, + MyCols.AskQuantityName => AskQuantity, + MyCols.AskMaxQuantityName => AskMaxQuantity, + MyCols.AskTradeQuantityName => AskTradeQuantity, + MyCols.AskInfluxQuantityName => AskInfluxQuantity, + MyCols.AskUnfilledQuantityName => AskUnfilledQuantity, + MyCols.TradeQuantityName => TradeQuantity, + MyCols.AvgTradePriceName => AvgTradePrice, + _ => this[Schema.ResolveIndices(colName)], + }; + + public int Id { get; } + + public DateTime? BidTime => bidPriceChangeRows().FirstOrDefault()?.Timestamp; + public DateTime? BidLastTime => reverseBidPriceChangeRows().FirstOrDefault()?.Timestamp; + public double BidLastQuantity => reverseBidPriceChangeRows().FirstOrDefault()?.BidQuantity ?? double.NaN; + public TimeSpan? BidPriceDuration => BidLastTime - BidTime; + /// + /// When the first trade occurs at the bid price + /// + public DateTime? BidTradeTime => + tradeTimeseries.Where(kv => kv.Value.AggressorSide == AggressorSide.Sell || kv.Value.Price <= BidPrice).TryGetFirst(out var kv) + ? kv.Value.Timestamp + : null; + public TimeSpan BidTimeTilTrade => + BidTradeTime.HasValue && BidTime.HasValue + ? BidTradeTime.Value - BidTime.Value + : TimeSpan.Zero; + public double BidPrice { get; } + public double BidQuantity { get; } + public double BidMaxQuantity { get; } + public double BidTradeQuantity => trades.Values.Where(t => t.AggressorSide == AggressorSide.Sell || t.Price <= BidPrice).Sum(t => t.Quantity); + public double BidInfluxQuantity => + bookTimeseries + .PairWithPrevious(false) + .Select(p => p.Current.Value.BidQuantity - p.Previous.Value.BidQuantity) + .Where(d => d > 0) + .Sum(); + public double BidUnfilledQuantity => BidQuantity + BidInfluxQuantity - BidTradeQuantity; + + public DateTime? LastTradeTime => + tradeTimeseries.Reverse().TryGetFirst(out var kv) + ? kv.Value.Timestamp + : null; + + public DateTime? OpenTimestamp { + get { + var rv = BidTime; + if (!rv.HasValue || (AskTime.HasValue && rv.Value > AskTime.Value)) { + rv = AskTime; + } + return rv; + } + } + public DateTime? CloseTimestamp { + get { + var rv = BidLastTime; + if (!rv.HasValue || (AskLastTime.HasValue && rv.Value > AskLastTime.Value)) { + rv = AskLastTime; + } + return rv; + } + } + public DateTime? AskTime => askPriceChangeRows().FirstOrDefault()?.Timestamp; + public DateTime? AskLastTime => reverseAskPriceChangeRows().FirstOrDefault()?.Timestamp; + public double AskLastQuantity => reverseAskPriceChangeRows().FirstOrDefault()?.AskQuantity ?? double.NaN; + public TimeSpan? AskPriceDuration => AskLastTime - AskTime; + public DateTime? AskTradeTime => + tradeTimeseries.Where(kv => kv.Value.Price >= AskPrice).TryGetFirst(out var kv) + ? kv.Value.Timestamp + : null; + public TimeSpan AskTimeTilTrade => + AskTradeTime.HasValue && AskTime.HasValue + ? AskTradeTime.Value - AskTime.Value + : TimeSpan.Zero; + public double AskPrice { get; } + public double AskQuantity { get; } + public double AskMaxQuantity { get; } + public double AskTradeQuantity => trades.Values.Where(t => t.AggressorSide == AggressorSide.Buy || t.Price >= AskPrice).Sum(t => t.Quantity); + public double AskInfluxQuantity => + bookTimeseries + .PairWithPrevious(false) + .Select(p => p.Current.Value.AskQuantity - p.Previous.Value.AskQuantity) + .Where(d => d > 0) + .Sum(); + public double AskUnfilledQuantity => AskQuantity + AskInfluxQuantity - AskTradeQuantity; + + public double TradeQuantity => trades.Values.Sum(t => t.Quantity); + public double AvgTradePrice { + get { + var tqty = TradeQuantity; + if (!tqty.is_real_positive()) { + return double.NaN; + } + return trades.Values.Sum(t => t.Price * t.Quantity) / tqty; + } + } + + IBlueTableSchema IBlueRow.schema => Schema; + + public BboPriceSummaryRow OnBook(IBlueRow row) => + OnBook(new BookRowData(row)); + + public BboPriceSummaryRow OnBook(BookRowData rowData) { + var booksBd = books.ToBuilder(); + var bookTimeseriesBd = bookTimeseries.ToBuilder(); + if (booksBd.TryGetValue(rowData.Id, out var oldRowData)) { + bookTimeseriesBd.Remove(new((oldRowData.Timestamp, oldRowData.Id), oldRowData)); + } + booksBd[rowData.Id] = rowData; + bookTimeseriesBd.Add(new((rowData.Timestamp, rowData.Id), rowData)); + return new(Id, booksBd.ToImmutable(), bookTimeseriesBd.ToImmutable(), trades, tradeTimeseries); + } + + public BboPriceSummaryRow OnTrade(IBlueRow row) => + OnTrade(new TradeRowData(row)); + + public BboPriceSummaryRow OnTrade(TradeRowData rowData) { + var tradesBd = trades.ToBuilder(); + var timeseriesBd = tradeTimeseries.ToBuilder(); + if (tradesBd.TryGetValue(rowData.Id, out var oldRowData)) { + timeseriesBd.Remove(new((oldRowData.Timestamp, oldRowData.Id), oldRowData)); + } + tradesBd[rowData.Id] = rowData; + timeseriesBd.Add(new((rowData.Timestamp, rowData.Id), rowData)); + return new(Id, books, bookTimeseries, tradesBd.ToImmutable(), timeseriesBd.ToImmutable()); + } + + IEnumerable bidPriceChangeRows() => + priceChangeRows(Side.Buy); + + IEnumerable reverseBidPriceChangeRows() => + reversePriceChangeRows(Side.Buy); + + IEnumerable askPriceChangeRows() => + priceChangeRows(Side.Sell); + + IEnumerable reverseAskPriceChangeRows() => + reversePriceChangeRows(Side.Sell); + + IEnumerable priceChangeRows(Side side) { + using var en = bookTimeseries.GetEnumerator(); + if (!en.MoveNext()) { + yield break; + } + var last = en.Current.Value; + yield return en.Current.Value; + while (en.MoveNext()) { + var (lastPrice, price) = side switch { + Side.Buy => (last.BidPrice, en.Current.Value.BidPrice), + Side.Sell => (last.AskPrice, en.Current.Value.AskPrice), + _ => throw new NotImplementedException(), + }; + if (price.Equals(lastPrice)) { + continue; + } + yield return en.Current.Value; + last = en.Current.Value; + } + } + + IEnumerable reversePriceChangeRows(Side side) { + using var en = bookTimeseries.Reverse().GetEnumerator(); + if (!en.MoveNext()) { + yield break; + } + BookRowData last = en.Current.Value; + while (en.MoveNext()) { + var (lastPrice, price) = side switch { + Side.Buy => (last.BidPrice, en.Current.Value.BidPrice), + Side.Sell => (last.AskPrice, en.Current.Value.AskPrice), + _ => throw new NotImplementedException(), + }; + if (!price.Equals(lastPrice)) { + yield return last; + } + last = en.Current.Value; + } + yield return last; + } + + class MyCols { + public const string IdName = "id"; + public const string TimestampName = WellKnownColumnNames.Timestamp; + public const string BidTimeName = "bid_time"; + public const string BidTimeTilTradeName = "bid_time_til_trade"; + public const string BidPriceName = WellKnownColumnNames.BidPrice; + public const string BidQuantityName = WellKnownColumnNames.BidQuantity; + public const string BidMaxQuantityName = "bid_max_size"; + public const string BidTradeQuantityName = "bid_trade_size"; + public const string BidInfluxQuantityName = "bid_size_influx"; + public const string BidUnfilledQuantityName = "bid_unfilled_size"; + public const string AskTimeName = "ask_time"; + public const string AskTimeTilTradeName = "ask_time_til_trade"; + public const string AskPriceName = WellKnownColumnNames.AskPrice; + public const string AskQuantityName = WellKnownColumnNames.AskQuantity; + public const string AskMaxQuantityName = "ask_max_size"; + public const string AskTradeQuantityName = "ask_trade_size"; + public const string AskInfluxQuantityName = "ask_influx_size"; + public const string AskUnfilledQuantityName = "ask_unfilled_size"; + public const string TradeQuantityName = "trade_size"; + public const string AvgTradePriceName = "avg_trade_price"; + + public const int IdIndex = 0; + public const int TimestampIndex = 1; + public const int BidTimeIndex = 2; + public const int BidTimeTilTradeIndex = 3; + public const int BidPriceIndex = 4; + public const int BidQuantityIndex = 5; + public const int BidMaxQuantityIndex = 6; + public const int BidTradeQuantityIndex = 7; + public const int BidInfluxQuantityIndex = 8; + public const int BidUnfilledQuantityIndex = 9; + public const int AskTimeIndex = 10; + public const int AskTimeTilTradeIndex = 11; + public const int AskPriceIndex = 12; + public const int AskQuantityIndex = 13; + public const int AskMaxQuantityIndex = 14; + public const int AskTradeQuantityIndex = 15; + public const int AskInfluxQuantityIndex = 16; + public const int AskUnfilledQuantityIndex = 17; + public const int TradeQuantityIndex = 18; + public const int AvgTradePriceIndex = 19; + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BboPriceSummaryStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BboPriceSummaryStream.cs new file mode 100644 index 000000000..dbc6c9119 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BboPriceSummaryStream.cs @@ -0,0 +1,169 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using bts; +using bts.api; +using bts.collections; + +using Bts.DataStreamCore; +using Bts.PubSub; + +using Spark.Core; +using Spark.Core.Collections; +using Spark.Core.Disposables; + +using System.Collections.Immutable; +using System.Diagnostics; + +[DataStreamUserRespresentation(false)] +public class BboPriceSummaryStream : ViewStream, IDisposable { + readonly IObservable bboStream; + readonly IObservable tradeStream; + readonly IWorkScheduler workScheduler; + readonly TimeSpan publishInterval; + IBlueRowCollection? lastFired; + + ImmutableSortedSet> bookTimeseries = + ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<(DateTime, int), BookRowData>(TupleComparer.Comparer())); + ImmutableSortedSet> tradeTimeseries = + ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<(DateTime, int), TradeRowData>(TupleComparer.Comparer())); + + int isDirty = 0; + IDisposable? sub; + + public BboPriceSummaryStream(IObservable bboStream, IObservable tradeStream, IWorkScheduler workScheduler, TimeSpan publishInterval) { + this.bboStream = bboStream; + this.tradeStream = tradeStream; + this.workScheduler = workScheduler; + this.publishInterval = publishInterval; + } + + ~BboPriceSummaryStream() => + Dispose(); + + public IEnumerable GetRows() { + int idCounter = 1; + var rows = bookTimeseries.AggregateConsecutiveGroups( + () => new BboPriceSummaryRow(idCounter++), + (r, kv) => r.OnBook(kv.Value), + kv => (kv.Value.BidPrice, kv.Value.AskPrice)) + .ToList(); + var tradeList = tradeTimeseries.ToList(); + int tradePos = 0; + foreach (var (r, next) in rows.PairWithPrevious()) { + if (r is null) { + continue; + } + var rOut = r; + if (rOut.BidTime.HasValue || rOut.AskTime.HasValue) { + var openTime = r.OpenTimestamp.assert_value(); + var closeTime = next.OpenTimestamp.assert_value(); + while (tradePos < tradeList.Count) { + var trade = tradeList[tradePos].Value; + if (trade.Timestamp < openTime) { + continue; + } + if (trade.Timestamp >= closeTime) { + break; + } + rOut = rOut.OnTrade(tradeList[tradePos].Value); + tradePos++; + } + } + yield return rOut; + } + } + + public void Dispose() { + GC.SuppressFinalize(this); + Disposable.dispose(ref sub); + } + + protected override void myOnFirstSubscriber() { + sub = Disposable.Create( + bboStream.Subscribe(OnBbo), + tradeStream.Subscribe(OnTrade), + workScheduler.Subscribe(Publish, publishInterval)); + + void OnBbo(IDataPacket dp) { + var bookTimeseriesBd = bookTimeseries.ToBuilder(); + foreach (IBlueRow r in dp.deletedRows) { + var rd = new BookRowData(r); + Log.assert(bookTimeseriesBd.Remove(new((rd.Timestamp, rd.Id), default!))); + } + foreach (IBlueRow r in dp.addedRows) { + var rd = new BookRowData(r); + Log.assert(bookTimeseriesBd.Add(new((rd.Timestamp, rd.Id), rd))); + } + bookTimeseries = bookTimeseriesBd.ToImmutable(); + Volatile.Write(ref isDirty, 1); + } + + void OnTrade(IDataPacket dp) { + var tradeTimeseriesBd = tradeTimeseries.ToBuilder(); + foreach (IBlueRow r in dp.deletedRows) { + var rd = new TradeRowData(r); + Log.assert(tradeTimeseriesBd.Remove(new((rd.Timestamp, rd.Id), rd))); + } + foreach (IBlueRow r in dp.addedRows) { + var rd = new TradeRowData(r); + Log.assert(tradeTimeseriesBd.Add(new((rd.Timestamp, rd.Id), rd))); + } + tradeTimeseries = tradeTimeseriesBd.ToImmutable(); + Volatile.Write(ref isDirty, 1); + } + + void Publish() { + if (Interlocked.CompareExchange(ref isDirty, 0, 1) == 0) { + return; + } + var sw = Stopwatch.StartNew(); + lastFired ??= new BlueTable(BboPriceSummaryRow.Schema); + var rows = GetRows().ToList(); + var table = new BlueTableListWrapper(BboPriceSummaryRow.Schema, rows.WrapSelector(r => (IBlueRow)r, null)); + var removes = new BlueTableListWrapper(BboPriceSummaryRow.Schema, lastFired.Cast().Where(r => !HasRow(table, r)).ToList()); + var adds = new BlueTableListWrapper(BboPriceSummaryRow.Schema, table.Cast().Where(r => !HasRow(lastFired, r)).ToList()); + lastFired = table; + sw.Stop(); + if (sw.Elapsed > TimeSpan.FromMilliseconds(20)) { + Log.DataFlow.warn($$"""[bbo_price_change_stream_slow_pub.json]{ "dur": "{{sw.ElapsedMilliseconds}}ms", "num_books": {{bookTimeseries.Count}} "num_trades": {{tradeTimeseries.Count}}, "ts": "{{DateTime.Now}}"}"""); + } + myFireEvent(new DataPacket(table, adds, removes)); + + bool HasRow(IBlueRowCollection? rows, IBlueRow row) { + if (rows is null) { + return false; + } + var id = row.Id(); + return rows.containsID(id) && BlueRowEqualityComparer.Equals(rows[id], row); + } + } + } + + protected override void myOnLastSubscriber() => + Disposable.dispose(ref sub); + + class BookSideRowData { + public BookSideRowData(Side side, IBlueRow row) { + Side = side; + Row = row; + Id = row.Id(); + Timestamp = row.GetDateTime(WellKnownColumns.Timestamp.Name).assert_value(); + var (priceCol, qtyCol) = side switch { + Side.Buy => (WellKnownColumns.BidPrice.Name, WellKnownColumns.BidQuantity.Name), + Side.Sell => (WellKnownColumns.AskPrice.Name, WellKnownColumns.AskQuantity.Name), + _ => throw new ArgumentException("Invalid side", nameof(side)), + }; + Price = row.GetFloat64(priceCol).assert_value(); + Quantity = row.GetFloat64(qtyCol).assert_value(); + } + + public Side Side { get; } + public IBlueRow Row { get; } + public int Id { get; } + public DateTime Timestamp { get; } + public double Price { get; } + public double Quantity { get; } + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BboPriceSummaryStreamFactory.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BboPriceSummaryStreamFactory.cs new file mode 100644 index 000000000..77548feb5 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BboPriceSummaryStreamFactory.cs @@ -0,0 +1,46 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using bts.BlueOrb; +using bts.collections; +using bts.md; + +using Bts.DataStream; +using Bts.DataStreamCore; +using Bts.PubSub; + +using System; + +public class BboPriceSummaryStreamFactory : DataStreamFactory { + static readonly VenueArgumentParser venueParser = VenueArgumentParser.Instance; + static readonly SecurityTypeArgumentParser secTypeParser = SecurityTypeArgumentParser.Instance; + + public BboPriceSummaryStreamFactory(MarketDataClient mdClient, IWorkScheduler workScheduler) { + this.Add( + "bboPriceSummary", + str => Create(mdClient, BboPriceSummaryStreamParams.FromStringArg(str, venueParser, secTypeParser), workScheduler), StringArgumentParser.Instance); + this.Add( + "bboPriceSummary", + venue => Create(mdClient, new() { Venue = venue }, workScheduler), venueParser); + this.Add( + "bboPriceSummary", + secType => Create(mdClient, new() { SecurityType = secType }, workScheduler), secTypeParser); + this.Add( + "bboPriceSummary", + (venue, secType) => Create(mdClient, new() { Venue = venue, SecurityType = secType }, workScheduler), venueParser, secTypeParser); + } + + public static ViewStream? Create(MarketDataClient mdClient, BboPriceSummaryStreamParams opts, IWorkScheduler workScheduler) { + var bboDataPackets = mdClient.books(opts.GetSecurityFilter()) + .AsDataPackets(opts.GetSchemaBuilder().BboAdapter()); + var bboTimeSeriesStream = new TimeSeriesStream(bboDataPackets.AsViewStream(), opts.BufferSize, WellKnownColumnNames.Timestamp); + var tradeSchemaBd = new TradeSchemaBuilder(); + var tradeSchema = tradeSchemaBd.ToSchema(); + var tradeAdapter = tradeSchemaBd.TradeInfoAdapter(); + var secFilter = opts.GetSecurityFilter(); + var tradeDataPackets = TradeStreamFactory.CreateDataPackets(mdClient.trades(secFilter), tradeSchema, tradeAdapter); + return new BboPriceSummaryStream(bboDataPackets, tradeDataPackets, workScheduler, TimeSpan.FromSeconds(1)); + } +} + diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BboPriceSummaryStreamParams.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BboPriceSummaryStreamParams.cs new file mode 100644 index 000000000..0d3182662 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BboPriceSummaryStreamParams.cs @@ -0,0 +1,9 @@ +namespace Bts.SharedStreams.MarketData; + +public class BboPriceSummaryStreamParams : BookStreamParams { + public int BufferSize { get; set; } + + public static new BboPriceSummaryStreamParams FromStringArg( + string str, VenueArgumentParser venueParser, SecurityTypeArgumentParser secTypeParser) => + FromStringArg(str, venueParser, secTypeParser); +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BookRowData.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BookRowData.cs new file mode 100644 index 000000000..a46420d6d --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BookRowData.cs @@ -0,0 +1,27 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using Bts.DataStreamCore; + +using Spark.Core; + +public class BookRowData { + public BookRowData(IBlueRow row) { + Row = row; + Id = row.Id(); + Timestamp = row.GetDateTime(WellKnownColumns.Timestamp.Name).assert_value(); + BidPrice = row.GetFloat64(WellKnownColumns.BidPrice.Name).assert_value(); + BidQuantity = row.GetFloat64(WellKnownColumns.BidQuantity.Name).assert_value(); + AskPrice = row.GetFloat64(WellKnownColumns.AskPrice.Name).assert_value(); + AskQuantity = row.GetFloat64(WellKnownColumns.AskQuantity.Name).assert_value(); + } + + public IBlueRow Row { get; } + public int Id { get; } + public DateTime Timestamp { get; } + public double BidPrice { get; } + public double BidQuantity { get; } + public double AskPrice { get; } + public double AskQuantity { get; } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BookSchemaBuilder.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BookSchemaBuilder.cs index 87ba6daef..4b1352ddc 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BookSchemaBuilder.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BookSchemaBuilder.cs @@ -5,7 +5,6 @@ using BCG; using bts.api; using bts.md; -using Bts.DataStream; using Bts.DataStreamCore; using Spark.Api.MarketData; diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BookStreamFactory.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BookStreamFactory.cs new file mode 100644 index 000000000..eae4d1fe7 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BookStreamFactory.cs @@ -0,0 +1,79 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using bts.api; +using bts.collections; + +using Bts.DataStream; +using Bts.DataStreamCore; +using Bts.PubSub; + +public class BookStreamFactory : DataStreamFactory { + readonly VenueArgumentParser venueParser = VenueArgumentParser.Instance; + readonly SecurityTypeArgumentParser secTypeParser = SecurityTypeArgumentParser.Instance; + readonly StringArgumentParser strParser = StringArgumentParser.Instance; + readonly MarketDataClient mdClient; + + public BookStreamFactory(MarketDataClient mdClient) { + this.mdClient = mdClient; + this.Add("book", createBook, venueParser); + this.Add("book", createBook, secTypeParser); + this.Add("book", createBook, strParser); + this.Add("book", createBook, venueParser, secTypeParser); + this.Add("book", createBook, venueParser, strParser); + this.Add("book", createBook, secTypeParser, strParser); + this.Add("bbo", createBbo, venueParser); + this.Add("bbo", createBbo, secTypeParser); + this.Add("bbo", createBbo, strParser); + this.Add("bbo", createBbo, venueParser, secTypeParser); + this.Add("bbo", createBbo, venueParser, strParser); + this.Add("bbo", createBbo, secTypeParser, strParser); + } + + ViewStream? createBook(Venue venue) => + createBook(new BookStreamParams() { Venue = venue }); + + ViewStream? createBook(SecurityType secType) => + createBook(new BookStreamParams() { SecurityType = secType }); + + ViewStream? createBook(Venue venue, SecurityType secType) => + createBook(new BookStreamParams() { Venue = venue, SecurityType = secType }); + + ViewStream? createBook(Venue venue, string nameOrAsset) => + createBook(new BookStreamParams() { Venue = venue, NameOrAsset = nameOrAsset }); + + ViewStream? createBook(SecurityType secType, string nameOrAsset) => + createBook(new BookStreamParams() { SecurityType = secType, NameOrAsset = nameOrAsset }); + + ViewStream? createBook(string str) => + createBook(BookStreamParams.FromStringArg(str, venueParser, secTypeParser)); + + ViewStream? createBook(BookStreamParams streamParams) => + mdClient.books(streamParams.GetSecurityFilter()) + .AsDataPackets(streamParams.GetSchemaBuilder().BookTupleAdapter()) + .AsViewStream(); + + ViewStream? createBbo(Venue venue) => + createBbo(new BookStreamParams() { Venue = venue }); + + ViewStream? createBbo(SecurityType secType) => + createBbo(new BookStreamParams() { SecurityType = secType }); + + ViewStream? createBbo(Venue venue, SecurityType secType) => + createBbo(new BookStreamParams() { Venue = venue, SecurityType = secType }); + + ViewStream? createBbo(Venue venue, string nameOrAsset) => + createBbo(new BookStreamParams() { Venue = venue, NameOrAsset = nameOrAsset }); + + ViewStream? createBbo(SecurityType secType, string nameOrAsset) => + createBbo(new BookStreamParams() { SecurityType = secType, NameOrAsset = nameOrAsset }); + + ViewStream? createBbo(string str) => + createBbo(BookStreamParams.FromStringArg(str, venueParser, secTypeParser)); + + ViewStream? createBbo(BookStreamParams streamParams) => + mdClient.books(streamParams.GetSecurityFilter()) + .AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter()) + .AsViewStream(); +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BookStreamParams.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BookStreamParams.cs index 040d8e0e2..7f6d87fde 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BookStreamParams.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BookStreamParams.cs @@ -2,6 +2,8 @@ namespace Bts.SharedStreams.MarketData; using bts.api; +using Bts.DataStreamCore; + using System.Text.RegularExpressions; public class BookStreamParams { @@ -10,7 +12,6 @@ public class BookStreamParams { public SecurityType? SecurityType { get; set; } public string? Name { get; set; } public string? NameOrAsset { get; set; } - public string? SecurityName { get; set; } public int NumLevels { get; set; } = 5; public bool ShowVenue { get; set; } = true; public bool ShowAsset { get; set; } = true; @@ -22,6 +23,25 @@ public class BookStreamParams { public bool ShowQuantity { get; set; } = true; public bool ShowNotional { get; set; } = true; + public static BookStreamParams FromStringArg(string str) => + FromStringArg(str, VenueArgumentParser.Instance, SecurityTypeArgumentParser.Instance); + + public static BookStreamParams FromStringArg(string str, VenueArgumentParser? venueParser, SecurityTypeArgumentParser? secTypeParser) => + FromStringArg(str, venueParser ?? VenueArgumentParser.Instance, secTypeParser ?? SecurityTypeArgumentParser.Instance); + + protected static T FromStringArg(string str, VenueArgumentParser venueParser, SecurityTypeArgumentParser secTypeParser) where T : BookStreamParams, new() { + var opts = new T(); + var arg = new StreamArgument(null, null, str); + if (venueParser.TryParse(arg, out var venue, out _)) { + opts.Venue = venue; + } else if (secTypeParser.TryParse(arg, out var secType, out _)) { + opts.SecurityType = secType; + } else { + opts.NameOrAsset = str; + } + return opts; + } + public Func GetSecurityFilter() { var nameRx = CreateRegex(Name); var nameOrAssetRx = CreateRegex(NameOrAsset); diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/InstrumentSchemaBuilder.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/InstrumentSchemaBuilder.cs new file mode 100644 index 000000000..23b05d68e --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/InstrumentSchemaBuilder.cs @@ -0,0 +1,84 @@ +namespace Bts.SharedStreams.MarketData; + +using bts.api; + +using Bts.DataStreamCore; + +public class InstrumentSchemaBuilder { + public bool ShowVenue { get; set; } = true; + public bool ShowAsset { get; set; } = true; + public bool ShowType { get; set; } = true; + public bool ShowName { get; set; } = true; + public bool ShowId { get; set; } = true; + public bool ShowExpiration { get; set; } = true; + public bool ShowCashPerPoint { get; set; } = true; + public bool ShowLegVenue { get; set; } + public bool ShowLegAsset { get; set; } + public bool ShowLegType { get; set; } + public bool ShowLegName { get; set; } = true; + public bool ShowLegId { get; set; } = true; + public bool ShowLegExpiration { get; set; } = true; + public bool ShowLegCashPerPoint { get; set; } + public int NumLegs { get; set; } = 2; + public string TableName { get; set; } = "Instrument"; + + public BlueTableSchemaAdapter SecurityAdapter() { + var adapter = new BlueTableSchemaAdapter(TableName); + adapter.AddColumn("id", r => (int)r.Id); + if (ShowVenue) { + adapter.AddColumn(WellKnownColumnNames.Venue, b => b.Exchange.ToString()); + } + if (ShowAsset) { + adapter.AddColumn(WellKnownColumnNames.Asset, b => b.Symbol); + } + if (ShowType) { + adapter.AddColumn(WellKnownColumnNames.SecurityType, b => b.security_type().ToString()); + } + if (ShowName) { + adapter.AddColumn(WellKnownColumnNames.SecurityName, b => b.ToString()); + } + if (ShowId) { + adapter.AddColumn(WellKnownColumnNames.SecurityId, b => b.Id); + } + if (ShowExpiration) { + adapter.AddColumn(WellKnownColumnNames.Expiration, b => b.security_expiration()); + } + if (ShowCashPerPoint) { + adapter.AddColumn(WellKnownColumns.CashPerPoint.Name, b => b.ContractSpec.CashPerPoint); + } + for (int i = 0; i < NumLegs; i++) { + if (ShowLegVenue) { + adapter.AddColumn(WellKnownColumnNames.LegVenue(i), Get(l => l.Security.Exchange.ToString(), null)); + } + if (ShowLegAsset) { + adapter.AddColumn(WellKnownColumnNames.LegAsset(i), Get(l => l.Security.Symbol, null)); + } + if (ShowLegType) { + adapter.AddColumn(WellKnownColumnNames.LegSecurityType(i), Get(l => l.Security.security_type().ToString(), null)); + } + if (ShowLegName) { + adapter.AddColumn(WellKnownColumnNames.LegSecurityName(i), Get(l => l.Security.ToString(), null)); + } + if (ShowLegId) { + adapter.AddColumn(WellKnownColumnNames.LegSecurityId(i), Get(l => (int)l.Security.Id, 0)); + } + if (ShowLegExpiration) { + adapter.AddColumn(WellKnownColumnNames.LegExpiration(i), Get(l => l.Security.security_expiration(), null)); + } + if (ShowLegCashPerPoint) { + adapter.AddColumn(WellKnownColumnNames.LegCashPerPoint(i), Get(l => l.Security.ContractSpec.CashPerPoint, double.NaN)); + } + + Func Get(Func getValue, T defaultValue) => + s => { + ContractLegImmutable? leg = s.ContractCase switch { + Security.ContractOneofCase.Spread => s.Spread.Legs.Skip(i).TryGetFirst(out var l) ? l : null, + Security.ContractOneofCase.Synthetic => s.Synthetic.Legs.Skip(i).TryGetFirst(out var l) ? l : null, + _ => null, + }; + return leg.HasValue ? getValue(leg.Value) : defaultValue; + }; + } + return adapter; + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/InstrumentStreamFactory.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/InstrumentStreamFactory.cs new file mode 100644 index 000000000..04a6dd058 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/InstrumentStreamFactory.cs @@ -0,0 +1,42 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using bts.api; +using bts.api.core; +using bts.BlueOrb; +using bts.md; + +using Bts.DataStream; +using Bts.DataStreamCore; + +using Microsoft.Extensions.Logging; + +using System.Reactive.Linq; + +class InstrumentStreamFactory : DataStreamFactory { + public InstrumentStreamFactory(MarketDataClient mdClient, ILogger logger) { + this.Add( + "instrument", + nameOrAsset => Create(mdClient, new() { NameOrAsset = nameOrAsset })); + this.Add( + "instrument", + (venue, nameOrAsset) => Create(mdClient, new() { Venue = venue, Asset = nameOrAsset })); + this.Add( + "instrument", + (venue, secType, nameOrAsset) => Create(mdClient, new() { Venue = venue, SecurityType = secType, NameOrAsset = nameOrAsset })); + this.Add("LiveFutureContractStream", tgt => CreateLiveFutureContract(mdClient, tgt), logger); + } + + public static ViewStream? Create(MarketDataClient mdClient, InstrumentStreamParams streamParams) { + var secFilter = streamParams.GetSecurityFilter(); + return mdClient.securities() + .Select(secs => secs.Where(kv => secFilter(kv.Value)).ToDictionary(kv => kv.Key, kv => kv.Value)) + .Table(s => s.Id) + .AsDataPackets(streamParams.GetSchemaBuilder().SecurityAdapter()) + .AsViewStream(); + } + + public static ViewStream? CreateLiveFutureContract(MarketDataClient mdClient, string target) => + new LiveFuturesContractStream(target ?? string.Empty, mdClient); +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/InstrumentStreamParams.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/InstrumentStreamParams.cs new file mode 100644 index 000000000..33a4f32eb --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/InstrumentStreamParams.cs @@ -0,0 +1,90 @@ +namespace Bts.SharedStreams.MarketData; + +using bts.api; + +using Bts.DataStreamCore; + +public class InstrumentStreamParams { + public Venue? Venue { get; set; } + public string? Asset { get; set; } + public SecurityType? SecurityType { get; set; } + public string? Name { get; set; } + public string? NameOrAsset { get; set; } + public int NumLegs { get; set; } = 5; + public bool ShowVenue { get; set; } = true; + public bool ShowType { get; set; } = true; + public bool ShowAsset { get; set; } = true; + public bool ShowName { get; set; } = true; + public bool ShowId { get; set; } = true; + public bool ShowExpiration { get; set; } + public bool ShowCashPerPoint { get; set; } = true; + public bool ShowLegVenue { get; set; } = true; + public bool ShowLegType { get; set; } = true; + public bool ShowLegAsset { get; set; } = true; + public bool ShowLegName { get; set; } = true; + public bool ShowLegId { get; set; } = true; + public bool ShowLegExpiration { get; set; } + public bool ShowLegCashPerPoint { get; set; } = true; + + public static InstrumentStreamParams FromStringArg(string str) => + FromStringArg(str, VenueArgumentParser.Instance, SecurityTypeArgumentParser.Instance); + + public static InstrumentStreamParams FromStringArg(string str, VenueArgumentParser? venueParser, SecurityTypeArgumentParser? secTypeParser) => + FromStringArg(str, venueParser ?? VenueArgumentParser.Instance, secTypeParser ?? SecurityTypeArgumentParser.Instance); + + protected static T FromStringArg(string str, VenueArgumentParser venueParser, SecurityTypeArgumentParser secTypeParser) where T : InstrumentStreamParams, new() { + var opts = new T(); + var arg = new StreamArgument(null, null, str); + if (venueParser.TryParse(arg, out var venue, out _)) { + opts.Venue = venue; + } else if (secTypeParser.TryParse(arg, out var secType, out _)) { + opts.SecurityType = secType; + } else { + opts.NameOrAsset = str; + } + return opts; + } + + + public Func GetSecurityFilter() { + return SecurityFilter; + + bool SecurityFilter(SecurityImmutable s) { + if (Venue.HasValue && Venue.Value != s.Exchange) { + return false; + } + if (SecurityType.HasValue && SecurityType != s.security_type()) { + return false; + } + if (Asset is not null && Asset != s.Symbol) { + return false; + } + if (Name is not null && Name != s.ToString()) { + return false; + } + if (NameOrAsset is not null && NameOrAsset != s.Symbol && NameOrAsset != s.ToString()) { + return false; + } + return true; + } + } + + public InstrumentSchemaBuilder GetSchemaBuilder() => + new() { + NumLegs = NumLegs, + ShowVenue = ShowVenue, + ShowType = ShowType, + ShowAsset = ShowAsset, + ShowName = ShowName, + ShowId = ShowId, + ShowExpiration = ShowExpiration, + ShowCashPerPoint = ShowCashPerPoint, + ShowLegVenue = ShowLegVenue, + ShowLegType = ShowLegType, + ShowLegAsset = ShowLegAsset, + ShowLegName = ShowLegName, + ShowLegId = ShowLegId, + ShowLegExpiration = ShowLegExpiration, + ShowLegCashPerPoint = ShowLegCashPerPoint, + }; +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesBookStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesBookStream.cs index 470e67c23..42cee34a5 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesBookStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesBookStream.cs @@ -1,13 +1,13 @@ namespace bts.BlueOrb { using BCG; - using System; + using bts.md; - [DataStreamUserRespresentation("LiveFuturesBooktream")] + [DataStreamUserRespresentation(false)] public class LiveFuturesBooktream : ViewStream { - public LiveFuturesBooktream(string future_target) { - ViewStream contract_str = new LiveFuturesContractStream(future_target); - ViewStream bbo_str = new BookStream(); + public LiveFuturesBooktream(string future_target, MarketDataClient mdClient) { + ViewStream contract_str = new LiveFuturesContractStream(future_target, mdClient); + ViewStream bbo_str = new BookStream(mdClient); ViewStream rs = new SimpleJoinStream(contract_str, "sec_key", bbo_str, "sec_key"); rs = new ProjectStream(rs, "left_sec:sec, left_sec_key:sec_key, bid_price, bid_sz:bid_size, ask_price, ask_sz:ask_size"); mySetRealSource(rs); diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesContractStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesContractStream.cs index 74b780cca..47119e999 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesContractStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesContractStream.cs @@ -4,13 +4,14 @@ using BCG; using bts.api; + using bts.md; [DataStreamUserRespresentation("LiveFuturesContractStream")] public class LiveFuturesContractStream : ViewStream { - public LiveFuturesContractStream(string target) { + public LiveFuturesContractStream(string target, MarketDataClient mdClient) { var fc = new ContractUtils.FutureContract(target); // temporary - ViewStream ins_str = new InstrumentsStream(); + ViewStream ins_str = new InstrumentsStream(mdClient); //new bts.BlueOrb.CSVReaderStream(@"C:\Temp\instruments.csv"); var rs = new LiveFuturesContractStream(ins_str, fc); mySetRealSource(rs); diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveTradeStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveTradeStream.cs deleted file mode 100644 index a38e5cce1..000000000 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveTradeStream.cs +++ /dev/null @@ -1,25 +0,0 @@ -namespace bts.BlueOrb { - using System; - - using BCG; - - using bts.api; - - [DataStreamUserRespresentation("LiveTradeStream")] - public class LiveTradeStream : ViewStream { - public LiveTradeStream(string target) - : this(SecurityType.Future, target) { } - public LiveTradeStream(SecurityType sec_type, string target) { - ViewStream rs = null; - switch (sec_type) { - case SecurityType.Future: - rs = new LiveFuturesTradeStream(target); - break; - default: - Log.assert(false); - break; - } - mySetRealSource(rs); - } - } -} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/SecurityTypeArgumentParser.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/SecurityTypeArgumentParser.cs index 6e309041f..0557cbe65 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/SecurityTypeArgumentParser.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/SecurityTypeArgumentParser.cs @@ -2,7 +2,17 @@ namespace Bts.SharedStreams.MarketData; using bts.api; -public static class SecurityTypeArgumentParser { +using Bts.DataStreamCore; + +public class SecurityTypeArgumentParser : IStreamArgumentParser { + public static readonly SecurityTypeArgumentParser Instance = new(); + + public bool CanParse(StreamArgument arg, Type outputType) => + TryParse(arg, out _, out _); + + public bool CanParse(StreamArgument arg) => + TryParse(arg, out _, out _); + public static bool TryParse(string? str, out SecurityType result) { if (str is null) { result = SecurityType.Unset; @@ -20,4 +30,23 @@ public static class SecurityTypeArgumentParser { }; return result != SecurityType.Unset; } + + public bool TryParse(StreamArgument arg, out SecurityType result, out string? error) { + error = null; + result = SecurityType.Unset; + var val = arg.Value; + if (val is SecurityType secType) { + result = secType; + return true; + } else if (val is string str) { + if (TryParse(str, out result)) { + return true; + } + error = $"Failed to parse SecurityType from string '{str}'"; + } else { + error = $"SecurityType parser received input of unexpected type '{val?.GetType()}'"; + } + return false; + } } + diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/StringArgumentParser.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/StringArgumentParser.cs new file mode 100644 index 000000000..4f211347d --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/StringArgumentParser.cs @@ -0,0 +1,25 @@ +namespace Bts.SharedStreams.MarketData; + +using Bts.DataStreamCore; + +using System.Diagnostics.CodeAnalysis; + +public class StringArgumentParser : IStreamArgumentParser { + public static readonly StringArgumentParser Instance = new(); + + public static bool TryParse(string? str, [NotNullWhen(true)]out string? v) { + v = str; + return v is not null; + } + + public bool CanParse(StreamArgument arg, Type outputType) => + TryParse(arg, out _, out _); + + public bool CanParse(StreamArgument arg) => + TryParse(arg, out _, out _); + + public bool TryParse(StreamArgument arg, [NotNullWhen(true)]out string? result, out string? error) { + error = null; + return TryParse(arg.Value as string ?? arg.Value?.ToString(), out result); + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/TradeRowData.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/TradeRowData.cs new file mode 100644 index 000000000..926ed8c0c --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/TradeRowData.cs @@ -0,0 +1,28 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using bts.co2; + +using Bts.DataStreamCore; +using Bts.SharedStreams; + +using Spark.Core; + +public class TradeRowData { + public TradeRowData(IBlueRow row) { + Row = row; + Id = row.Id(); + Timestamp = row.GetDateTime(WellKnownColumns.Timestamp.Name).assert_value(); + AggressorSide = row.GetEnum("aggressor_side") ?? AggressorSide.None; + Price = row.GetFloat64(WellKnownColumns.Price.Name).assert_value(); + Quantity = row.GetFloat64(WellKnownColumns.Quantity.Name).assert_value(); + } + + public IBlueRow Row { get; } + public int Id { get; } + public DateTime Timestamp { get; } + public AggressorSide AggressorSide { get; } + public double Price { get; } + public double Quantity { get; } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/TradeSchemaBuilder.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/TradeSchemaBuilder.cs new file mode 100644 index 000000000..0ad0f2cc5 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/TradeSchemaBuilder.cs @@ -0,0 +1,92 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using bts.api; + +using Bts.DataStreamCore; + +using mbts.spark; +using mbts.utils; + +public class TradeSchemaBuilder { + public bool ShowNotional { get; set; } + public bool ShowQuantity { get; set; } = true; + public bool ShowPrice { get; set; } = true; + public bool ShowVenue { get; set; } + public bool ShowAsset { get; set; } = true; + public bool ShowSecurityType { get; set; } + public bool ShowSecurityName { get; set; } + public bool ShowSecurityId { get; set; } = true; + public bool ShowCashPerPoint { get; set; } + public bool ShowTimestamp { get; set; } = true; + public string TableName { get; set; } = "Trade"; + + public IBlueTableSchema ToSchema() { + var bd = new BlueTableSchemaBuilder(TableName) { + ("id", typeof(int)) + }; + if (ShowTimestamp) { + bd.Add(WellKnownColumns.Timestamp); + } + if (ShowVenue) { + bd.Add(WellKnownColumns.Venue); + } + if (ShowSecurityType) { + bd.Add(WellKnownColumns.SecurityType); + } + if (ShowAsset) { + bd.Add(WellKnownColumns.Asset); + } + if (ShowSecurityName) { + bd.Add(WellKnownColumns.SecurityName); + } + if (ShowSecurityId) { + bd.Add(WellKnownColumns.SecurityId); + } + if (ShowCashPerPoint) { + bd.Add(WellKnownColumns.CashPerPoint); + } + if (ShowPrice) { + bd.Add(WellKnownColumns.Price); + } + if (ShowQuantity) { + bd.Add(WellKnownColumns.Quantity); + } + return bd.ToSchema(); + } + + public BlueTableSchemaAdapter TradeInfoAdapter() { + var adapter = new BlueTableSchemaAdapter(TableName); + adapter.AddColumn("id", r => r.RecordId); + if (ShowTimestamp) { + adapter.AddColumn(WellKnownColumns.Timestamp.Name, b => new LLDateTime((ulong)b.trade.Time).to_datetime()); + } + if (ShowVenue) { + adapter.AddColumn(WellKnownColumns.Venue.Name, b => b.security.Exchange.ToString()); + } + if (ShowSecurityType) { + adapter.AddColumn(WellKnownColumns.SecurityType.Name, b => b.security.security_type().ToString()); + } + if (ShowAsset) { + adapter.AddColumn(WellKnownColumns.Asset.Name, b => b.security.Symbol); + } + if (ShowSecurityName) { + adapter.AddColumn(WellKnownColumns.SecurityName.Name, b => b.security.ToString()); + } + if (ShowSecurityId) { + adapter.AddColumn(WellKnownColumns.SecurityId.Name, b => b.security.Id); + } + if (ShowCashPerPoint) { + adapter.AddColumn(WellKnownColumns.CashPerPoint.Name, b => b.security.ContractSpec.CashPerPoint); + } + if (ShowPrice) { + adapter.AddColumn(WellKnownColumns.Price.Name, b => b.trade.Mantissa * Math.Pow(10, b.trade.Exponent)); + } + if (ShowQuantity) { + // TODO: Make trade size a float (it's still uint) + adapter.AddColumn(WellKnownColumns.Quantity.Name, b => b.trade.Size); + } + return adapter; + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/TradeStreamFactory.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/TradeStreamFactory.cs new file mode 100644 index 000000000..d7e931241 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/TradeStreamFactory.cs @@ -0,0 +1,52 @@ +namespace Bts.SharedStreams.MarketData; + +using BCG; + +using bts.api; +using bts.collections; +using bts.md; + +using Bts.DataStream; +using Bts.DataStreamCore; +using Bts.PubSub; + +using mbts.spark; + +using System; +using System.Reactive.Linq; + +public class TradeStreamFactory : DataStreamFactory { + readonly VenueArgumentParser venueParser = VenueArgumentParser.Instance; + readonly SecurityTypeArgumentParser secTypeParser = SecurityTypeArgumentParser.Instance; + readonly StringArgumentParser strParser = StringArgumentParser.Instance; + readonly MarketDataClient mdClient; + + public TradeStreamFactory(MarketDataClient mdClient) { + this.mdClient = mdClient; + this.Add("trade", venue => Create(new() { Venue = venue }), venueParser); + this.Add("trade", secType => Create(new() { SecurityType = secType }), secTypeParser); + this.Add("trade", str => Create(TradeStreamParams.FromStringArg(str, venueParser, secTypeParser)), strParser); + this.Add("trade", (venue, secType) => Create(new() { Venue = venue, SecurityType = secType }), venueParser, secTypeParser); + this.Add("trade", (venue, nameOrAsset) => Create(new() { Venue = venue, NameOrAsset = nameOrAsset }), venueParser, strParser); + this.Add("trade", (secType, nameOrAsset) => Create(new() { SecurityType = secType, NameOrAsset = nameOrAsset }), secTypeParser, strParser); + + ViewStream? Create(TradeStreamParams opts) { + var secFilter = opts.GetSecurityFilter(); + var schemaBd = opts.GetSchemaBuilder(); + var schema = schemaBd.ToSchema(); + var adapter = schemaBd.TradeInfoAdapter(); + return CreateDataPackets(mdClient.trades(secFilter), schema, adapter).AsViewStream(); + } + } + + public static IObservable CreateDataPackets(IObservable> trades, IBlueTableSchema tradeSchema, BlueTableSchemaAdapter tradeAdapter) => + trades.Scan((IDataPacket)null!, (dp, trades) => { + var rowList = trades.WrapSelector(t => (IBlueRow)new BlueRowAdapter(tradeAdapter, t), null); + var lastTable = dp?.newRows as BlueTableImmutable ?? new BlueTableImmutable(tradeSchema); + var bd = lastTable.ToDataPacketBuilder(); + foreach (var t in trades) { + bd.AddOrUpdate(new BlueRowAdapter(tradeAdapter, t)); + } + return bd.ToDataPacket(); + }); +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/TradeStreamParams.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/TradeStreamParams.cs new file mode 100644 index 000000000..e459bd5ee --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/TradeStreamParams.cs @@ -0,0 +1,73 @@ +namespace Bts.SharedStreams.MarketData; + +using bts.api; + +using Bts.DataStreamCore; + +public class TradeStreamParams { + public Venue? Venue { get; set; } + public string? Asset { get; set; } + public SecurityType? SecurityType { get; set; } + public string? Name { get; set; } + public string? NameOrAsset { get; set; } + public bool ShowVenue { get; set; } = true; + public bool ShowAsset { get; set; } = true; + public bool ShowSecurityType { get; set; } = true; + public bool ShowSecurityName { get; set; } = true; + public bool ShowPrice { get; set; } = true; + public bool ShowQuantity { get; set; } = true; + public bool ShowNotional { get; set; } = true; + public bool ShowCashPerPoint { get; set; } = true; + + public static TradeStreamParams FromStringArg(string str, VenueArgumentParser venueParser, SecurityTypeArgumentParser secTypeParser) => + FromStringArg(str, venueParser, secTypeParser); + + public Func GetSecurityFilter() { + return SecurityFilter; + + bool SecurityFilter(SecurityImmutable s) { + if (Venue.HasValue && Venue.Value != s.Exchange) { + return false; + } + if (SecurityType.HasValue && SecurityType != s.security_type()) { + return false; + } + if (Asset is not null && Asset != s.Symbol) { + return false; + } + if (Name is not null && Name != s.ToString()) { + return false; + } + if (NameOrAsset is not null && NameOrAsset != s.Symbol && NameOrAsset != s.ToString()) { + return false; + } + return true; + } + } + + public TradeSchemaBuilder GetSchemaBuilder() => + new() { + ShowVenue = ShowVenue, + ShowSecurityType = ShowSecurityType, + ShowAsset = ShowAsset, + ShowSecurityName = ShowSecurityName, + ShowSecurityId = true, + ShowPrice = ShowPrice, + ShowQuantity = ShowQuantity, + ShowNotional = ShowNotional, + ShowCashPerPoint = ShowCashPerPoint, + }; + + protected static T FromStringArg(string str, VenueArgumentParser venueParser, SecurityTypeArgumentParser secTypeParser) where T : TradeStreamParams, new() { + var opts = new T(); + var arg = new StreamArgument(null, null, str); + if (venueParser.TryParse(arg, out var venue, out _)) { + opts.Venue = venue; + } else if (secTypeParser.TryParse(arg, out var secType, out _)) { + opts.SecurityType = secType; + } else { + opts.NameOrAsset = str; + } + return opts; + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/VenueArgumentParser.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/VenueArgumentParser.cs index 6b6642cc8..2ea4b2a5d 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/VenueArgumentParser.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/VenueArgumentParser.cs @@ -2,7 +2,11 @@ namespace Bts.SharedStreams.MarketData; using bts.api; -public static class VenueArgumentParser { +using Bts.DataStreamCore; + +public class VenueArgumentParser : IStreamArgumentParser { + public static readonly VenueArgumentParser Instance = new(); + public static bool TryParse(string? str, out Venue result) { if (str is null) { result = Venue.Unset; @@ -18,4 +22,28 @@ public static class VenueArgumentParser { }; return result != Venue.Unset; } + + public bool CanParse(StreamArgument arg, Type outputType) => + TryParse(arg, out _, out _); + + public bool CanParse(StreamArgument arg) => + TryParse(arg, out _, out _); + + public bool TryParse(StreamArgument arg, out Venue result, out string? error) { + error = null; + result = Venue.Unset; + var val = arg.Value; + if (val is Venue venue) { + result = venue; + return true; + } else if (val is string str) { + if (TryParse(str, out result)) { + return true; + } + error = $"Failed to parse venue from string '{str}'"; + } else { + error = $"Venue parser received input of unexpected type '{val?.GetType()}'"; + } + return false; + } } diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbScoreStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbScoreStream.cs index 5d847ea59..b2217468b 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbScoreStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbScoreStream.cs @@ -1,15 +1,16 @@ namespace bts.BlueOrb { - using bts.SilverPine; using BCG; using System; + using bts.md; - [DataStreamUserRespresentation("SpatialArbScoreStream")] + [DataStreamUserRespresentation(false)] + /// This stream is now created by public class SpatialArbScoreStream : ViewStream { - public SpatialArbScoreStream(int window_sz_secs, int sample_period_sz) - : this(TimeSpan.FromSeconds(window_sz_secs), TimeSpan.FromSeconds(sample_period_sz), ViewStreamDatabase.instance) { } + public SpatialArbScoreStream(MarketDataClient mdClient, int window_sz_secs, int sample_period_sz) + : this(mdClient, TimeSpan.FromSeconds(window_sz_secs), TimeSpan.FromSeconds(sample_period_sz), ViewStreamDatabase.instance) { } - public SpatialArbScoreStream(TimeSpan window_sz, TimeSpan sample_period, ViewStreamDatabase vsd) { - ViewStream rs = new SpatialArbStatsStream(window_sz, sample_period, vsd); + public SpatialArbScoreStream(MarketDataClient mdClient, TimeSpan window_sz, TimeSpan sample_period, ViewStreamDatabase vsd) { + ViewStream rs = new SpatialArbStatsStream(mdClient, window_sz, sample_period, vsd); rs = new ThrottleStream(rs, TimeSpan.FromSeconds(1), vsd); rs = new SortStream(rs, "-n_trade_ticks"); mySetRealSource(rs); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs index 2985d8e81..c06b8ef25 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs @@ -1,17 +1,15 @@ namespace bts.BlueOrb { using BCG; - using SharpCompress.Common.Rar; + using bts.md; using System; using System.Collections.Generic; - using System.Linq; - using System.Text; - using System.Threading.Tasks; // For all the pairs generated by SpatialFuturePairsStream, this stream // generates the statistics, by appending together a SpreadStatsStream - [DataStreamUserRespresentation("SpatialArbStatsStream")] + /// This stream is now created by + [DataStreamUserRespresentation(false)] public class SpatialArbStatsStream : ViewStream{ private ViewStream _spatial_pairs_stream; private ViewStreamDatabase _vsd; @@ -20,14 +18,14 @@ private ViewStream _bbo_stream; private readonly TimeSpan _window_sz, _sampling_period_sz; - public SpatialArbStatsStream(int window_sz_seconds, int sampling_period_sz_seconds) - : this(TimeSpan.FromSeconds(window_sz_seconds), TimeSpan.FromSeconds(sampling_period_sz_seconds), ViewStreamDatabase.instance) { } + public SpatialArbStatsStream(MarketDataClient mdClient, int window_sz_seconds, int sampling_period_sz_seconds) + : this(mdClient, TimeSpan.FromSeconds(window_sz_seconds), TimeSpan.FromSeconds(sampling_period_sz_seconds), ViewStreamDatabase.instance) { } - public SpatialArbStatsStream(TimeSpan window_sz, TimeSpan sampling_period_sz, ViewStreamDatabase vsd) { + public SpatialArbStatsStream(MarketDataClient mdClient, TimeSpan window_sz, TimeSpan sampling_period_sz, ViewStreamDatabase vsd) { _vsd = vsd; _window_sz = window_sz; _sampling_period_sz = sampling_period_sz; - _spatial_pairs_stream = new SpatialFuturePairsStream(vsd); + _spatial_pairs_stream = new SpatialFuturePairsStream(mdClient, vsd); _bbo_stream = new BBOStream(vsd); } diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs index 4c59e7cea..084e92631 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs @@ -2,23 +2,22 @@ using BCG; using bts.api; + using bts.md; using System; using System.Collections.Generic; - using System.Linq; - using System.Text; - using System.Threading.Tasks; - [DataStreamUserRespresentation("SpatialFuturePairsStream")] + /// This stream is now created by + [DataStreamUserRespresentation(false)] public class SpatialFuturePairsStream : ViewStream { private ViewStream _main_stream; private IBlueTableSchema _schema; private HashSet _key_map = new HashSet(); - public SpatialFuturePairsStream() - : this(ViewStreamDatabase.instance) { } + public SpatialFuturePairsStream(MarketDataClient mdClient) + : this(mdClient, ViewStreamDatabase.instance) { } - public SpatialFuturePairsStream(ViewStreamDatabase vsd) { - _main_stream = new InstrumentsStream(vsd); + public SpatialFuturePairsStream(MarketDataClient mdClient, ViewStreamDatabase vsd) { + _main_stream = new InstrumentsStream(mdClient, vsd); _main_stream = new FilterStream(_main_stream, "sec_type='Future'"); _schema = new BlueTableSchema("SpatialFutures", new ColumnInfo("id", 0, TypeCode.Int32)); _schema = _schema.add_column(TypeCode.String, "s1"); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs index b83c616be..54eca6034 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs @@ -101,7 +101,7 @@ namespace bts.BlueOrb { } } - [DataStreamUserRespresentation("bbo")] + [DataStreamUserRespresentation(false)] public class BBOStream : ViewStream, IWorker { private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500); private DateTime _last_poll_time = DateTime.MinValue; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs index 24b3f24a6..1b25cd42a 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs @@ -18,7 +18,7 @@ namespace bts.BlueOrb{ using SecurityKey = System.UInt32; using System.Diagnostics; - [DataStreamUserRespresentation("books")] + [DataStreamUserRespresentation(false)] public class BookStream : ViewStream, IWorker { private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500); private DateTime _last_poll_time = DateTime.MinValue; @@ -39,14 +39,13 @@ namespace bts.BlueOrb{ IDisposable? subscription; // TODO: add cons for symbol, exchange etc - public BookStream() - : this(ViewStreamDatabase.instance) { - } + public BookStream(MarketDataClient mdClient) + : this(mdClient, ViewStreamDatabase.instance) { } - public BookStream(ViewStreamDatabase vsd) { + public BookStream(MarketDataClient mdClient, ViewStreamDatabase vsd) { bts.Log.Gen.info("BookStream.cons"); _vsd = vsd; - _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5))); + _md_client = mdClient; } protected override void myOnFirstSubscriber() { diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs index 4a83b24cb..6fb5c4a4c 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs @@ -43,14 +43,13 @@ namespace bts.BlueOrb { IDisposable? subscription; // TODO: add cons for symbol, exchange etc - public InstrumentsStream() - : this(ViewStreamDatabase.instance) { - } + public InstrumentsStream(MarketDataClient mdClient) + : this(mdClient, ViewStreamDatabase.instance) { } - public InstrumentsStream(ViewStreamDatabase vsd) { + public InstrumentsStream(MarketDataClient mdClient, ViewStreamDatabase vsd) { bts.Log.Gen.info("InstrumentsStream.cons"); _vsd = vsd; - _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5))); + _md_client = mdClient; } protected override void myOnFirstSubscriber() { diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs index ae5523fb3..4713fe802 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs @@ -213,7 +213,14 @@ namespace bts.BlueOrb { } - - + internal OHLCVStream(ViewStream source, OHLCVBarSize bar_size) { + ViewStream rs = source; + rs = new SchemaGuesserStream(rs); + if (bar_size == OHLCVBarSize.ONE_DAY) { + rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime); + } + rs = new NewProjectStream(rs, "symbol:contract"); + mySetRealSource(rs); + } } } diff --git a/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLStream.cs b/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLStream.cs index 330dfc309..cf40a0cd4 100644 --- a/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLStream.cs +++ b/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLStream.cs @@ -1,19 +1,23 @@ namespace bts.BlueOrb { using BCG; + using Bts.DataStreamCore; + using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; - [DataStreamUserRespresentation("nl")] + [DataStreamUserRespresentation(false)] public class BlueOrbNLStream : ViewStream { private readonly string _nl_prompt; + readonly IDataStreamFactory streamFactory; private ViewStream _nl_to_dsl_stream, _dsl_stream; private IDictionary _dp_metadata = new Dictionary(); - public BlueOrbNLStream(string nl_prompt) { + public BlueOrbNLStream(string nl_prompt, IDataStreamFactory streamFactory) { _nl_prompt = nl_prompt; + this.streamFactory = streamFactory; } protected override void myOnFirstSubscriber() { @@ -27,7 +31,7 @@ string dsl = r.get("response"); _dp_metadata["dsl"] = dsl; Log.Gen.info($"NLStream._handle_dsl_callback dsl: {dsl}"); - _dsl_stream = new DSLStream(dsl); + _dsl_stream = new DSLStream(dsl, streamFactory); _dsl_stream.onChange += _handle_data_cb; } diff --git a/src/DataStreamSuite/SharedStreams/NL/NaturalLanguageStreamFactory.cs b/src/DataStreamSuite/SharedStreams/NL/NaturalLanguageStreamFactory.cs new file mode 100644 index 000000000..62cc39de2 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/NL/NaturalLanguageStreamFactory.cs @@ -0,0 +1,30 @@ +namespace Bts.SharedStreams.NaturalLanguage; + +using BCG; + +using bts.BlueOrb; + +using Bts.DataStreamCore; + +using Microsoft.Extensions.Logging; + +public class NaturalLanguageStreamFactory : DataStreamFactory { + IDataStreamFactory? compositeFactory; + + public NaturalLanguageStreamFactory(ILogger logger) { + const string streamName = "NaturalLanguageStream"; + + this.Add(streamName, Create); + + ViewStream? Create(string prompt) { + if (compositeFactory is null) { + logger.LogInvalidParams(streamName, "'compositeFactory' is null."); + return null; + } + return new BlueOrbNLStream(prompt, compositeFactory); + } + } + + public void OnCompositeFactory(IDataStreamFactory compositeFactory) => + this.compositeFactory = compositeFactory; +} diff --git a/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs b/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs new file mode 100644 index 000000000..3a4c9c685 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs @@ -0,0 +1,24 @@ +namespace Bts.SharedStreams.Services; + +using Bts.DataStreamCore; +using Bts.SharedStreams.MarketData; +using Bts.SharedStreams.NaturalLanguage; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +public static class SharedStreamsServices { + public static IServiceCollection AddMarketDataStreams(this IServiceCollection services) => + services.addStreamFactory() + .addStreamFactory() + .addStreamFactory() + .addStreamFactory(); + + public static IServiceCollection AddNaturalLanguageStreams(this IServiceCollection services, IConfigurationSection configurationSection) => + services.AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()); + + static IServiceCollection addStreamFactory(this IServiceCollection services) where T : class, IDataStreamFactory => + services.AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()); +} diff --git a/src/client/spark_api/MarketData/BookTuple.cs b/src/client/spark_api/MarketData/BookTuple.cs index db7074cae..11c035931 100644 --- a/src/client/spark_api/MarketData/BookTuple.cs +++ b/src/client/spark_api/MarketData/BookTuple.cs @@ -3,8 +3,10 @@ namespace Spark.Api.MarketData { using bts.api; using bts.api.domain; using bts.md; + using bts.utils; using System; + using System.Linq; public readonly struct BookTuple : IEquatable, IJsonFormatter { readonly TickDisplaySettingsImmutable? _quantity_display; @@ -19,9 +21,13 @@ namespace Spark.Api.MarketData { _quantity_display = quantity_display; } + public static BookTuple Empty(SecurityImmutable security) => + new(security, Book.empty(security.Id)); + public SecurityImmutable security { get; } public uint security_id => security.Id; public Book book { get; } + public Bbo bbo { get { (double bidp, double bidq) = book.bids.Count > 0 ? (book.bids[0].price, book.bids[0].size) : (double.NaN, double.NaN); @@ -33,6 +39,8 @@ namespace Spark.Api.MarketData { public TickDisplaySettingsImmutable quantity_display => _quantity_display ?? Security.Defaults.quantity_display_settingsi; + public DateTime? Timestamp { get; init; } + public bool Equals(BookTuple other) => security.Equals(other.security) && book.Equals(other.book); @@ -52,4 +60,15 @@ namespace Spark.Api.MarketData { public static bool operator !=(BookTuple left, BookTuple right) => !(left == right); } + + public static class BookTupleExtensions { + public static Book NotionalBook(this BookTuple bookTuple) => + bookTuple.book.ToNotional(bookTuple.security.ContractSpec.CashPerPoint); + + public static Book ToNotional(this Book book, double cpp) => + new( + book.security_id, + book.bids.Select(b => new PriceSize((float)(b.size * cpp * b.price), b.price)).ToList(), + book.asks.Select(a => new PriceSize((float)(a.size * cpp * a.price), a.price)).ToList()); + } } diff --git a/src/client/spark_api/MarketData/MarketDataClient.cs b/src/client/spark_api/MarketData/MarketDataClient.cs index ccc5120ae..3cae89791 100644 --- a/src/client/spark_api/MarketData/MarketDataClient.cs +++ b/src/client/spark_api/MarketData/MarketDataClient.cs @@ -60,7 +60,7 @@ namespace bts.md { } public IObservable> books(SecurityCriteria? criteria = null) => - books(criteria is null ? null : + books(criteria is null || !criteria.is_defined() ? null : new Func(s => criteria.includes(s))); public IObservable> books(Func? filter) { @@ -99,10 +99,15 @@ namespace bts.md { } } - public IObservable> trades(TradesSubscriptionFilter filter) { - var sec_mds = filter.security_criteria == null ? + public IObservable> trades(TradesSubscriptionFilter filter) => + trades(filter.security_criteria is null || !filter.security_criteria.is_defined() + ? null + : filter.security_criteria.includes); + + public IObservable> trades(Func? secFilter) { + IObservable> sec_mds = secFilter is null ? _source.subscribe_securities() : - filter.security_criteria.subscribe_md_map(_source.subscribe_securities()); + _source.subscribe_securities().Select(secs => secs.Where(kv => secFilter(kv.Value.security)).ToDictionary()); return TradesSource.create(_source.consumer, sec_mds, _poll_interval); } diff --git a/src/client/spark_api/MarketData/PriceSize.cs b/src/client/spark_api/MarketData/PriceSize.cs index 42d04c3af..5a59d6acd 100644 --- a/src/client/spark_api/MarketData/PriceSize.cs +++ b/src/client/spark_api/MarketData/PriceSize.cs @@ -2,14 +2,12 @@ namespace bts.md { using System; using Size = System.Single; - public class PriceSize - { + public class PriceSize { public Size size { get; } public Size direct_size { get; } public Size implied_size { get; } public double price { get; } - public PriceSize(Size sz, Size direct_sz, Size implied_sz, double prx) - { + public PriceSize(Size sz, Size direct_sz, Size implied_sz, double prx) { size = sz; direct_size = direct_sz; implied_size = implied_sz; diff --git a/src/client/spark_api/MarketData/SyntheticBook.cs b/src/client/spark_api/MarketData/SyntheticBook.cs new file mode 100644 index 000000000..185d4783b --- /dev/null +++ b/src/client/spark_api/MarketData/SyntheticBook.cs @@ -0,0 +1,26 @@ +#nullable enable +using bts.api; + +using Spark.Api.MarketData; + +using System.Collections.Generic; + +public readonly struct SyntheticBook { + public SyntheticBook( + uint securityId, IReadOnlyList bids, IReadOnlyList asks) { + SecurityId = securityId; + Bids = bids; + Asks = asks; + } + + public uint SecurityId { get; } + public IReadOnlyList Bids { get; } + public IReadOnlyList Asks { get; } + + public SyntheticBookLevel this[Side side, int level] => + side switch { + Side.Buy => level < Bids.Count ? Bids[level] : new(double.NaN, double.NaN, double.NaN), + Side.Sell => level < Asks.Count ? Asks[level] : new(double.NaN, double.NaN, double.NaN), + _ => throw new KeyNotFoundException("Invalid side"), + }; +} diff --git a/src/client/spark_api/MarketData/SyntheticBookCalculator.cs b/src/client/spark_api/MarketData/SyntheticBookCalculator.cs index fd70cc6fd..da87f931e 100644 --- a/src/client/spark_api/MarketData/SyntheticBookCalculator.cs +++ b/src/client/spark_api/MarketData/SyntheticBookCalculator.cs @@ -3,6 +3,7 @@ namespace Spark.Api.MarketData; using bts; using bts.api; +using bts.collections; using bts.md; using bts.utils; @@ -18,7 +19,7 @@ public static class SyntheticBookCalculator { GetSyntheticDepth(legDepths.Select(GetBidLegParams).ToList()).Select(ToPriceSize).ToList(), GetSyntheticDepth(legDepths.Select(GetAskLegParams).ToList()).Select(ToPriceSize).ToList()); - PriceSize ToPriceSize(DepthLevel level) => + static PriceSize ToPriceSize(DepthLevel level) => new((float)level.Quantity, level.Price); (double Quantity, double TickSize, Book Book) GetBidLegParams((ContractLegImmutable, Book) ld) => @@ -28,6 +29,22 @@ public static class SyntheticBookCalculator { (-ld.Item1.signed_quantity(), ld.Item1.Security.ContractSpec.LowTick, ld.Item2); } + public static Book GetSyntheticBook(uint securityId, IEnumerable<(double Quantity, double TickSize, Book Book)> legDepths) { + return new( + securityId, + GetSyntheticDepth(legDepths.ToList()).Select(ToPriceSize).ToList(), + GetSyntheticDepth(legDepths.Select(l => (-l.Quantity, l.TickSize, l.Book)).ToList()).Select(ToPriceSize).ToList()); + + static PriceSize ToPriceSize(DepthLevel level) => + new((float)level.Quantity, level.Price); + } + + public static SyntheticBook GetSyntheticNotionalBook(uint securityId, IEnumerable<(double Quantity, double TickSize, double CashPerPoint, Book Book)> legDepths) => + new( + securityId, + GetSyntheticNotionalDepth(legDepths.ToList()).ToList(), + GetSyntheticNotionalDepth(legDepths.Select(l => (-l.Quantity, l.TickSize, l.CashPerPoint, l.Book)).ToList()).ToList()); + public static List GetSyntheticDepth(IEnumerable<(ContractLegImmutable, Book)> legDepths) { return GetSyntheticDepth( legDepths.Select(getLegDepthConsumer).ToList()); @@ -37,15 +54,18 @@ public static class SyntheticBookCalculator { Log.assert(leg.Quantity.is_real(), "Leg quantity must be real"); Log.assert(leg.Security.HasContractSpec && leg.Security.ContractSpec.LowTick.is_real_positive(), "Leg security LowTick must be real and positive"); - return LegDepthConsumer.Create(leg.Quantity, leg.Security.ContractSpec.LowTick); + var depthLevels = (leg.Side == Side.Buy ? book.bids : book.asks) + .Select(lvl => new DepthLevel(lvl.price, lvl.size)) + .ToArray(); + return LegDepthConsumer.Create(leg.Quantity, leg.Security.ContractSpec.LowTick, depthLevels); } } public static List GetSyntheticDepth(IEnumerable<(double Quantity, double TickSize, Book Book)> legDepths) { return GetSyntheticDepth( - legDepths.Select(getLegDepthConsumer).ToList()); + legDepths.Select(GetLegDepthConsumer).ToList()); - LegDepthConsumer getLegDepthConsumer((double Quantity, double TickSize, Book Book) legDepth) { + LegDepthConsumer GetLegDepthConsumer((double Quantity, double TickSize, Book Book) legDepth) { var (qty, tickSz, bk) = legDepth; Log.assert(qty.is_real(), "Leg quantity must be real"); Log.assert(tickSz.is_real_positive(), @@ -54,6 +74,29 @@ public static class SyntheticBookCalculator { } } + public static List GetSyntheticNotionalDepth(IEnumerable<(double Quantity, double TickSize, double CashPerPoint, Book Book)> legDepths) { + var legs = legDepths.Select(InitializeLegDepth).ToList(); + if (legs.Count == 0) { + return new(0); + } + return legs.Skip(1).Aggregate(legs[0], AggregateLegDepth).Depth.Levels; + + SyntheticLegDepthConsumer InitializeLegDepth((double Quantity, double TickSize, double CashPerPoint, Book Book) legDepth) { + var (qty, tickSz, cashPerPoint, bk) = legDepth; + Log.assert(qty.is_real(), "Leg quantity must be real"); + Log.assert(tickSz.is_real_positive(), + "Leg tickSz must be real and positive"); + var levels = (qty > 0 ? bk.bids : bk.asks).ToArray(l => new SyntheticBookLevel(l.price, l.size, l.NotionalValue(cashPerPoint))); + return SyntheticLegDepthConsumer.Create(qty, tickSz, levels); + } + + SyntheticLegDepthConsumer AggregateLegDepth(SyntheticLegDepthConsumer l1, SyntheticLegDepthConsumer l2) => + new(new(getSyntheticNotionalDepth(l1, l2).ToList())) { + LegQuantity = 1, + QuantityTick = MathUtils.lcm(l1.QuantityTick, l2.QuantityTick), + }; + } + public static List GetSyntheticDepth(List legs) { if (legs.Count == 0) { return new(0); @@ -90,6 +133,39 @@ public static class SyntheticBookCalculator { } } + static IEnumerable getSyntheticNotionalDepth( + SyntheticLegDepthConsumer leg1, SyntheticLegDepthConsumer leg2) { + while (!leg1.Depth.IsDone && !leg2.Depth.IsDone) { + if (!leg1.LegQuantity.is_real() || !leg2.LegQuantity.is_real()) { + throw new ArgumentException("Invalid leg quantities"); + } + double leg2PerLeg1 = (leg2.LegQuantity / leg1.LegQuantity).abs(); + double leg1PerLeg2 = (leg1.LegQuantity / leg2.LegQuantity).abs(); + double avail1 = leg1.Depth.AvailableQuantity(1); + double avail2 = leg2.Depth.AvailableQuantity(1); + (double take1, double take2) = (avail1 / leg1PerLeg2) < (avail2 / leg2PerLeg1) + ? (avail1, avail1 * leg2PerLeg1) + : (avail2 * leg1PerLeg2, avail2); + var (qty1, cost1, notional1) = leg1.Depth.ConsumeQuantity(take1); + var (qty2, cost2, notional2) = leg2.Depth.ConsumeQuantity(take2); + Log.assert(qty2 / qty1 == leg2PerLeg1); + var price1 = cost1 / qty1; + var price2 = cost2 / qty2; + price1 *= leg1.LegQuantity; + price2 *= leg2.LegQuantity; + if (leg1.LegQuantity < 0) { + notional1 = -notional1; + } + if (leg2.LegQuantity < 0) { + notional2 = -notional2; + } + yield return new( + Price: price1 + price2, + Quantity: qty1, + NotionalValue: notional1 + notional2); + } + } + public record DepthLevel(double Price, double Quantity) { public override string ToString() { return $"{Price} ({Quantity})"; @@ -173,6 +249,7 @@ public static class SyntheticBookCalculator { public double LegQuantity { get; set; } = 1; public double QuantityTick { get; set; } = 1; + public double CashPerPoint { get; set; } = 1; public DepthConsumer Depth { get; } = new(); public static LegDepthConsumer Create(double legQuantity, double quantityTick, params DepthLevel[] levels) => diff --git a/src/client/spark_api/MarketData/SyntheticBookLevel.cs b/src/client/spark_api/MarketData/SyntheticBookLevel.cs new file mode 100644 index 000000000..6eaf2da58 --- /dev/null +++ b/src/client/spark_api/MarketData/SyntheticBookLevel.cs @@ -0,0 +1,12 @@ +#nullable enable + +namespace Spark.Api.MarketData; + +/// +/// Like a normal BookLevel, but includes the NotionalValue, as that could only +/// otherwise be computed with the leg quantities and cash per point. +/// +/// +/// +/// +public readonly record struct SyntheticBookLevel(double Price, double Quantity, double NotionalValue); diff --git a/src/client/spark_api/MarketData/SyntheticDepthConsumer.cs b/src/client/spark_api/MarketData/SyntheticDepthConsumer.cs new file mode 100644 index 000000000..2d2c973ab --- /dev/null +++ b/src/client/spark_api/MarketData/SyntheticDepthConsumer.cs @@ -0,0 +1,73 @@ +#nullable enable +namespace Spark.Api.MarketData; + +using System; +using System.Collections.Generic; +using System.Linq; + +public class SyntheticDepthConsumer { + public SyntheticDepthConsumer() { } + public SyntheticDepthConsumer(params SyntheticBookLevel[] levels) : this(levels.AsEnumerable()) { } + public SyntheticDepthConsumer(IEnumerable levels) => + Levels.AddRange(levels); + + public int Pos { get; private set; } = 0; + public double CurrLevelConsumed { get; private set; } = 0; + public List Levels { get; } = new(); + + public bool IsDone => + Pos == Levels.Count || (Pos == Levels.Count - 1 && Levels.Last().Quantity == CurrLevelConsumed); + + public double AvailableQuantity(int levelLimit) { + int end = Math.Min(Pos + levelLimit, Levels.Count); + double rv = 0; + for (int i = Pos; i < end; i++) { + double levelQty = Levels[i].Quantity; + if (i == Pos) { + levelQty -= CurrLevelConsumed; + } + rv += levelQty; + } + return rv; + } + + public (double Qty, double Price, double Notional) ConsumeQuantity(double quantity) { + double consumed = 0; + double consumedNotional = 0; + double price = 0; + for (; Pos < Levels.Count; Pos++) { + var level = Levels[Pos]; + double levelQty = level.Quantity - CurrLevelConsumed; + var rem = quantity - consumed; + if (levelQty > rem) { + CurrLevelConsumed += rem; + consumed = quantity; + var remCost = rem * Levels[Pos].Price; + price += remCost; + consumedNotional += Levels[Pos].NotionalValue * rem / Levels[Pos].Quantity; + break; + } + consumed += levelQty; + price += levelQty * Levels[Pos].Price; + consumedNotional += Levels[Pos].NotionalValue * levelQty / Levels[Pos].Quantity; + CurrLevelConsumed = 0; + } + return (consumed, price, consumedNotional); + } +} + +public class SyntheticLegDepthConsumer { + public SyntheticLegDepthConsumer() : this(new()) { } + public SyntheticLegDepthConsumer(SyntheticDepthConsumer depth) => + Depth = depth; + + public double LegQuantity { get; set; } = 1; + public double QuantityTick { get; set; } = 1; + public SyntheticDepthConsumer Depth { get; } = new(); + + public static SyntheticLegDepthConsumer Create(double legQuantity, double quantityTick, params SyntheticBookLevel[] levels) => + new(new(levels)) { + LegQuantity = legQuantity, + QuantityTick = quantityTick, + }; +} diff --git a/src/client/spark_api/MarketData/TradeInfo.cs b/src/client/spark_api/MarketData/TradeInfo.cs index 2f57ab8b3..aecdf75ef 100644 --- a/src/client/spark_api/MarketData/TradeInfo.cs +++ b/src/client/spark_api/MarketData/TradeInfo.cs @@ -12,16 +12,19 @@ namespace mbts.spark { public readonly TradeMessageImmutable trade; public readonly GreeksTuple? greeks_tuple; public readonly double volume; + public readonly int RecordId; public TradeInfo( SecurityMD security_md, TradeMessageImmutable trade, GreeksTuple? greeks_tuple = null, - double volume = double.NaN) { + double volume = double.NaN, + int recordId = 0) { this.security_md = security_md; this.trade = trade; this.greeks_tuple = greeks_tuple; this.volume = volume; + RecordId = recordId; } public SecurityImmutable security => security_md.security; @@ -32,7 +35,8 @@ namespace mbts.spark { price = obj.trade.price(), timestamp = obj.trade.Time, quantity = obj.volume, - agg_side = obj.trade.AggressorSide.ToString() + agg_side = obj.trade.AggressorSide.ToString(), + rec_id = obj.RecordId, }; } @@ -42,11 +46,13 @@ namespace mbts.spark { SecurityMD? security_md = null, TradeMessageImmutable? trade = null, Optional greeks_tuple = default, - double? volume = null) => + double? volume = null, + int? recordId = null) => new( security_md ?? trade_info.security_md, trade ?? trade_info.trade, greeks_tuple.value_or(trade_info.greeks_tuple), - volume ?? trade_info.volume); + volume ?? trade_info.volume, + recordId ?? trade_info.RecordId); } } diff --git a/src/client/spark_api/data/SecurityCriteriaExtensions.cs b/src/client/spark_api/data/SecurityCriteriaExtensions.cs index bf1a11ba8..ee3e6ac94 100644 --- a/src/client/spark_api/data/SecurityCriteriaExtensions.cs +++ b/src/client/spark_api/data/SecurityCriteriaExtensions.cs @@ -108,10 +108,19 @@ namespace mbts.spark { security_criteria.is_defined() && security_criteria.clauses.empty_if_null().Any(c => c.includes(security)); + public static bool includes( + this SecurityCriteria security_criteria, (Venue? Venue, SecurityType? Type, string? Symbol) tup) => + security_criteria.is_defined() && + security_criteria.clauses.empty_if_null().Any(c => c.includes(tup)); + public static bool includes( this SecurityCriteriaClause clause, SecurityImmutable security) => _to_predicates(clause).AnyAndAll(p => p(security)); + public static bool includes( + this SecurityCriteriaClause clause, (Venue? Venue, SecurityType? Type, string? Symbol) tup) => + _to_symbol_predicates(clause).AnyAndAll(p => p(tup)); + public static bool is_subset_of( this SecurityCriteriaClause clause, SecurityCriteriaClause other) => // exchanges @@ -178,6 +187,10 @@ namespace mbts.spark { ) ); + public static bool includes(this SecurityCriteria security_criteria, (Venue Venue, SecurityType Type, string? symbol) sec_tup) => + security_criteria.is_defined() && + security_criteria.clauses.empty_if_null().Any(c => c.includes(sec_tup)); + public static SecurityCriteriaClause with(this SecurityCriteriaClause clause, Venue[]? exchanges = null, SecurityType[]? security_types = null, @@ -472,5 +485,17 @@ namespace mbts.spark { yield return sec => clause.names.Contains(sec.ToString()); } } + + static IEnumerable> _to_symbol_predicates(SecurityCriteriaClause clause) { + if (!clause.exchanges.is_null_or_empty()) { + yield return tup => !tup.Venue.HasValue || clause.exchanges.Contains(tup.Venue.Value); + } + if (!clause.symbols.is_null_or_empty()) { + yield return tup => tup.Symbol is null || clause.symbols.Contains(tup.Symbol); + } + if (!clause.security_types.is_null_or_empty()) { + yield return tup => !tup.Type.HasValue || clause.security_types.Contains(tup.Type.Value); + } + } } } diff --git a/src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs b/src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs index 174bf1c44..8faa5a50f 100644 --- a/src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs +++ b/src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs @@ -308,6 +308,21 @@ namespace System.Collections.Generic { return source.All(hs.Add); } + public static IEnumerable<(T? Previous, T Current)> PairWithPrevious(this IEnumerable source, bool includeZeroIndex = true) { + using var en = source.GetEnumerator(); + if (!en.MoveNext()) { + yield break; + } + var prev = en.Current; + if (includeZeroIndex) { + yield return (default, prev); + } + while (en.MoveNext()) { + yield return (prev, en.Current); + prev = en.Current; + } + } + /// /// Checks if implements or . If so, the resultant Count is returned, otherwise null (will not enumerate through any items). /// @@ -426,7 +441,7 @@ namespace System.Collections.Generic { } public static bool TryGetFirst( - this IEnumerable source, Func predicate, out T? result, out int index) { + this IEnumerable source, Func predicate, [MaybeNullWhen(false)]out T result, out int index) { if (source == null) throw new ArgumentNullException(nameof(source)); index = 0; foreach (T element in source) { @@ -498,6 +513,88 @@ namespace System.Collections.Generic { return false; } + public static bool TryGetExactCount(this IEnumerable source, [MaybeNullWhen(false)]out T value1) { + ArgumentNullException.ThrowIfNull(source); + using var en = source.GetEnumerator(); + value1 = default; + return tryGetNext(en, out value1) + && !en.MoveNext(); + } + + public static bool TryGetExactCount(this IEnumerable source, [MaybeNullWhen(false)]out T value1, [MaybeNullWhen(false)] out T value2) { + ArgumentNullException.ThrowIfNull(source); + using var en = source.GetEnumerator(); + value1 = default; + value2 = default; + return tryGetNext(en, out value1) + && tryGetNext(en, out value2) + && !en.MoveNext(); + } + + public static bool TryGetExactCount(this IEnumerable source, [MaybeNullWhen(false)]out T value1, [MaybeNullWhen(false)] out T value2, [MaybeNullWhen(false)] out T value3) { + ArgumentNullException.ThrowIfNull(source); + using var en = source.GetEnumerator(); + value1 = default; + value2 = default; + value3 = default; + return tryGetNext(en, out value1) + && tryGetNext(en, out value2) + && tryGetNext(en, out value3) + && !en.MoveNext(); + } + + public static bool TryGetExactCount( + this IEnumerable source, + [MaybeNullWhen(false)] out T value1, + [MaybeNullWhen(false)] out T value2, + [MaybeNullWhen(false)] out T value3, + [MaybeNullWhen(false)] out T value4 + ) { + ArgumentNullException.ThrowIfNull(source); + using var en = source.GetEnumerator(); + value1 = default; + value2 = default; + value3 = default; + value4 = default; + return tryGetNext(en, out value1) + && tryGetNext(en, out value2) + && tryGetNext(en, out value3) + && tryGetNext(en, out value4) + && !en.MoveNext(); + } + + public static bool TryGetExactCount( + this IEnumerable source, + [MaybeNullWhen(false)] out T value1, + [MaybeNullWhen(false)] out T value2, + [MaybeNullWhen(false)] out T value3, + [MaybeNullWhen(false)] out T value4, + [MaybeNullWhen(false)] out T value5 + ) { + ArgumentNullException.ThrowIfNull(source); + using var en = source.GetEnumerator(); + value1 = default; + value2 = default; + value3 = default; + value4 = default; + value5 = default; + return tryGetNext(en, out value1) + && tryGetNext(en, out value2) + && tryGetNext(en, out value3) + && tryGetNext(en, out value4) + && tryGetNext(en, out value5) + && !en.MoveNext(); + } + + static bool tryGetNext(IEnumerator enumerator, [MaybeNullWhen(false)]out T result) { + while (enumerator.MoveNext()) { + result = enumerator.Current; + return true; + } + result = default; + return false; + } + public static bool TryFindIndex(this IEnumerable source, T value, out int index) => TryFindIndex(source, value, EqualityComparer.Default, out index); diff --git a/src/client/spark_core/Json/JsonMultiLineString.cs b/src/client/spark_core/Json/JsonMultiLineString.cs new file mode 100644 index 000000000..9c0321018 --- /dev/null +++ b/src/client/spark_core/Json/JsonMultiLineString.cs @@ -0,0 +1,9 @@ +#nullable enable +namespace Spark.Core.Json; + +using System.Text.Json.Serialization; + +[JsonConverter(typeof(JsonMultiLineStringConverter))] +public class JsonMultiLineString { + public string? Value { get; set; } +} diff --git a/src/client/spark_core/Json/JsonMultiLineStringConverter.cs b/src/client/spark_core/Json/JsonMultiLineStringConverter.cs new file mode 100644 index 000000000..fc786364f --- /dev/null +++ b/src/client/spark_core/Json/JsonMultiLineStringConverter.cs @@ -0,0 +1,39 @@ +#nullable enable +namespace Spark.Core.Json; + +using System; + +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +// Reads normal strings and arrays of lines +public class JsonMultiLineStringConverter : JsonConverter { + public override JsonMultiLineString? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (reader.ReadString(out var str)) { + return new() { Value = str }; + } + if (reader.ReadArray(out var array)) { + return new() { Value = string.Join('\n', array.Select(v => v?.AsString())) }; + } + throw new JsonException("Failed to parse multi-line string"); + } + + public override void Write(Utf8JsonWriter writer, JsonMultiLineString value, JsonSerializerOptions options) => + writer.WriteStringValue(value.Value); +} + +public class JsonMultiLineStringValueConverter : JsonConverter { + public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (reader.ReadString(out var str)) { + return str; + } + if (reader.ReadArray(out var array)) { + return str; + } + throw new JsonException("Failed to parse multi-line string"); + } + + public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) => + writer.WriteStringValue(value); +} diff --git a/src/client/spark_core/Update/TableUpdate.cs b/src/client/spark_core/Update/TableUpdate.cs index 888ddcfda..ff6b971e1 100644 --- a/src/client/spark_core/Update/TableUpdate.cs +++ b/src/client/spark_core/Update/TableUpdate.cs @@ -3,6 +3,7 @@ namespace bts.api.core { using System; using System.Collections.Generic; using System.Collections.Immutable; + using System.Diagnostics.CodeAnalysis; using System.Linq; public class TableUpdate where TKey : IEquatable { @@ -277,6 +278,17 @@ namespace bts.api.core { _removed[kv.Key] = kv.Value; } + public bool TryGetValue(TKey key, [MaybeNullWhen(false)]out T result) { + if(_added.TryGetValue(key, out result)) { + return true; + } + if (_changes.TryGetValue(key, out var ch)) { + result = ch.New; + return true; + } + return !_removed.ContainsKey(key) && _state.state.TryGetValue(key, out result); + } + /// /// Creates a TableUpdate instance from the current state /// diff --git a/src/client/spark_core/utils/EnumHelper.cs b/src/client/spark_core/utils/EnumHelper.cs index b1ca947ce..d570e3cf2 100644 --- a/src/client/spark_core/utils/EnumHelper.cs +++ b/src/client/spark_core/utils/EnumHelper.cs @@ -68,6 +68,9 @@ namespace mbts.ui.utility { return Activator.CreateInstance(enum_type); } + public static T? TryParse(string value, bool ignoreCase = true) where T : struct, Enum => + Enum.TryParse(value, ignoreCase, out T res) ? res : null; + public static T try_convert(object value) { var numeric_type = Enum.GetUnderlyingType(typeof(T)); var numeric_value = Convert.ChangeType(value, numeric_type); diff --git a/src/client/spark_websocket/Core/WebsocketDataSync.cs b/src/client/spark_websocket/Core/WebsocketDataSync.cs index f1700356f..44232e579 100644 --- a/src/client/spark_websocket/Core/WebsocketDataSync.cs +++ b/src/client/spark_websocket/Core/WebsocketDataSync.cs @@ -16,6 +16,7 @@ namespace bts.websocket_core { using bts.api.position; using bts.api.synthetic; using Spark.Api.Services; + using Bts.DataStreamCore; /// @@ -26,10 +27,13 @@ namespace bts.websocket_core { readonly AdminService adminService; readonly Spark.Api.IServiceProvider _service_ctx; readonly IUserDataPermissions userDataPermissions; + readonly IDataStreamFactory streamFactory; - public WebsocketDataSync(Spark.Api.IServiceProvider service_ctx, IUserDataPermissions userDataPermissions) { + public WebsocketDataSync(Spark.Api.IServiceProvider service_ctx, MarketDataClient mdClient, IUserDataPermissions userDataPermissions, IEnumerable streamFactories) { _service_ctx = service_ctx; this.userDataPermissions = userDataPermissions; + this.mdClient = mdClient; + streamFactory = DataStreamFactory.Create(streamFactories); adminService = service_ctx.GetService(); } @@ -328,6 +332,7 @@ namespace bts.websocket_core { req, Md_Client, Orders_Client, + streamFactory, obj => send_message(obj, req.client_id), out hdlr, out status)) { diff --git a/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs b/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs index 5d6794eab..7fab17507 100644 --- a/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs +++ b/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs @@ -224,6 +224,7 @@ namespace bts.websocket_core { } static bool TryGetDataStream( WebsocketRequest req, + IDataStreamFactory streamFactory, [NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(false)] out string error ) { @@ -253,7 +254,7 @@ namespace bts.websocket_core { } Exception? exception = null; try { - stream = exp.getView(new Dictionary(0)); + stream = exp.getView(new Dictionary(0), streamFactory); } catch (Exception ex) { exception = ex; } @@ -306,6 +307,7 @@ namespace bts.websocket_core { WebsocketRequest req, MarketDataClient md, OrdersClient oc, + IDataStreamFactory streamFactory, Action sm, [NotNullWhen(true)] out WebsocketSubscriptionHandler? hdlr, out string status @@ -372,7 +374,7 @@ namespace bts.websocket_core { } case SUBSCRIPTION_TYPE.DATA_STREAM: { // TODO: make this nullable - if (!TryGetDataStream(req, out var filter, out status)) { + if (!TryGetDataStream(req, streamFactory, out var filter, out status)) { return false; } sub = GetDataStreamSubscriber(filter, id.Value.Val);