diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index ceeb06ae8..02a9e91bd 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -25,6 +25,12 @@ "commands": [ "fake" ] + }, + "dotnet-t4": { + "version": "3.0.0", + "commands": [ + "t4" + ] } } } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Misc/DSLStream.cs b/src/DataStreamSuite/DataStream/Misc/DSLStream.cs index dea082576..5ed3fdbcf 100644 --- a/src/DataStreamSuite/DataStream/Misc/DSLStream.cs +++ b/src/DataStreamSuite/DataStream/Misc/DSLStream.cs @@ -3,23 +3,24 @@ namespace bts.BlueOrb { using BCG; using System.Collections.Generic; using Bts.DataStreamExpressions; + using Bts.DataStreamCore; // All this does is take in a dsl expression, parse it and fire the data that come out of running it. // So for example, if an expresison is passed [DataStreamUserRespresentation("dsl")] public class DSLStream : ViewStream { private readonly string _dsl; - public DSLStream(string dsl) { + public DSLStream(string dsl, IDataStreamFactory streamFactory) { Log.Gen.info($"DSLStream.cons dsl: {dsl}"); _dsl = dsl; - ViewStream rs = build_stream(dsl) ?? new LiteralStream("error", $"msg:invalid dsl"); + var rs = build_stream(streamFactory, dsl) ?? new LiteralStream("error", $"msg:invalid dsl"); mySetRealSource(rs); } - static ViewStream? build_stream(string stream_expression) { + static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) { try { var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); - var rv = exp.getView(new Dictionary(0)); + var rv = exp.getView(new Dictionary(0), streamFactory); return rv; } catch (Exception ex) { diff --git a/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs b/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs index be8b833a2..d129236f0 100644 --- a/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs +++ b/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs @@ -35,12 +35,12 @@ namespace BCG { view_publisher = view_publisher_; } } - static ViewStream? build_stream(string stream_expression) + static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) { try { var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); - var rv = exp.getView(new Dictionary(0)); + var rv = exp.getView(new Dictionary(0), streamFactory); return rv; } catch (Exception ex) { @@ -52,15 +52,17 @@ namespace BCG { _stream_pub_map = new Dictionary(); readonly IPubTableBroker _broker; readonly IManagedSubscriber _sub; + readonly IDataStreamFactory streamFactory; readonly object syncRoot = new(); IDisposable? cloudStreamsSub; IDisposable? remoteRequestsSub; bool isDisposed; - public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) { + public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker, IDataStreamFactory streamFactory) { _sub = managedSubscriber; _broker = pubTableBroker; + this.streamFactory = streamFactory; } public Task StartAsync(CancellationToken cancellationToken) { @@ -103,7 +105,11 @@ namespace BCG { private void _spawn_remote_stream(string name, string stream_expression, bool persist) { bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}"); - ViewStream stream = build_stream(stream_expression); + var stream = build_stream(streamFactory, stream_expression); + if (stream is null) { + Log.DataFlow.error($"Failed to spawn remote stream. name: {name}; expression: {stream_expression}"); + return; + } _expose_to_ws(name, stream); if (persist) { string tempDir = Path.GetTempPath(); diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs index 71e4fccd7..ae7387cab 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs @@ -19,6 +19,8 @@ using Newtonsoft.Json; using BCG; using System.Runtime.CompilerServices; using bts; +using Spark.Core; +using System.Numerics; namespace bts { @@ -91,13 +93,13 @@ namespace BCG { return BlueDataUtility.get_row_datetime_value(r, col_no, err_value); } - public static double? GetDouble(this IBlueRow r, int colNo) => - GetDouble(r[colNo]); + public static double? GetFloat64(this IBlueRow r, int colNo) => + GetFloat64(r[colNo]); - public static double? GetDouble(this IBlueRow r, string colName) => - GetDouble(r[colName]); + public static double? GetFloat64(this IBlueRow r, string colName) => + GetFloat64(r[colName]); - public static double? GetDouble(object obj) { + public static double? GetFloat64(object obj) { if (obj is double d) { return d; } @@ -125,6 +127,127 @@ namespace BCG { return null; } + public static int? GetInt32(this IBlueRow r, int colNo) => + GetInt32(r[colNo]); + + public static int? GetInt32(this IBlueRow r, string colName) => + GetInt32(r[colName]); + + public static int? GetInt32(object obj) { + checked { + if (obj is double d) { + return (int)d; + } + if (obj is int i32) { + return i32; + } + if (obj is long i64) { + return (int)i64; + } + if (obj is string s) { + return double.TryParse(s, out d) ? (int)d : null; + } + if (obj is uint u32) { + return (int)u32; + } + if (obj is ulong u64) { + return (int)u64; + } + if (obj is short i16) { + return i16; + } + if (obj is ushort u16) { + return u16; + } + } + return null; + } + + public static long? GetInt64(this IBlueRow r, int colNo) => + GetInt64(r[colNo]); + + public static long? GetInt64(this IBlueRow r, string colName) => + GetInt64(r[colName]); + + public static long? GetInt64(object obj) { + checked { + if (obj is double d) { + return (long)d; + } + if (obj is int i32) { + return i32; + } + if (obj is long i64) { + return i64; + } + if (obj is string s) { + return double.TryParse(s, out d) ? (long)d : null; + } + if (obj is uint u32) { + return u32; + } + if (obj is ulong u64) { + return (long)u64; + } + if (obj is short i16) { + return i16; + } + if (obj is ushort u16) { + return u16; + } + } + return null; + } + + public static DateTime? GetUnixEpochDateTime(this IBlueRow row, string colName) => + GetUnixEpochDateTime(row[colName]); + + public static DateTime? GetUnixEpochDateTime(this IBlueRow row, int colNo) => + GetUnixEpochDateTime(row[colNo]); + + public static DateTime? GetUnixEpochDateTime(object obj) { + var tsNanos = GetInt64(obj); + return tsNanos.HasValue ? DateTime.UnixEpoch.Add(new TimeSpan(tsNanos.Value / 100)) : null; + } + + public static DateTime? GetDateTime(object obj) { + var res = _get_datetime_value(obj, DateTime.MinValue); + if (res == DateTime.MinValue) { + return null; + } + return res; + } + + public static object GetEnum(Type type, object o) { + try { + if (o is null || o == DBNull.Value) { + return null; + } else if (o is string str) { + return Enum.TryParse(type, str, true, out var res) ? res : null; + } else if (o is int || o is uint || o is long || o is ulong || o is short || o is ushort) { + return Convert.ChangeType(o, type); + } + } catch { + } + return null; + } + + public static T? GetEnum(object o) where T : struct, Enum { + try { + if (o is null || o == DBNull.Value) { + return null; + } else if (o is string str) { + return Enum.TryParse(str, true, out T res) ? res : null; + } else if (o is int || o is short || o is ushort || o is uint || o is long || o is ulong) + return (T)o; + else if (o is T t) { + return t; + } + } catch { + } + return null; + } + public static T get_enum(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct { return BlueDataUtility.get_row_enum_value(r, col_name, err_value); } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowStore.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowStore.cs new file mode 100644 index 000000000..28ab1b035 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowStore.cs @@ -0,0 +1,311 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using mbts.utils; + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; + +public class BlueRowStore : IBlueRow, IReadOnlyDictionary { + readonly string? name; + IBlueTableSchema? _schema; + + public BlueRowStore() { } + + public BlueRowStore( + int id, + (string Name, T1 Value) item1, + (string Name, T2 Value) item2, + (string Name, T3 Value) item3, + (string Name, T4 Value) item4, + (string Name, T5 Value) item5, + (string Name, T6 Value) item6 + ) { + Id = id; + Item1 = item1; + Item2 = item2; + Item3 = item3; + Item4 = item4; + Item5 = item5; + Item6 = item6; + } + + public int Id { get; set; } + public (string Name, T1 Value) Item1 { get; set; } + public (string Name, T2 Value) Item2 { get; set; } + public (string Name, T3 Value) Item3 { get; set; } + public (string Name, T4 Value) Item4 { get; set; } + public (string Name, T5 Value) Item5 { get; set; } + public (string Name, T6 Value) Item6 { get; set; } + + public int Count => 5; + public IReadOnlyList Keys => new KeyList(this); + public IReadOnlyList Values => new ValueList(this); + + public IBlueTableSchema schema => + _schema ??= new BlueTableSchema(name, new ColumnInfo[]{ + new("id", 0, typeof(int)), + new(Item1.Name, 1, typeof(T1)), + new(Item2.Name, 2, typeof(T2)), + new(Item3.Name, 3, typeof(T3)), + new(Item4.Name, 4, typeof(T4)), + new(Item5.Name, 5, typeof(T5)), + new(Item6.Name, 6, typeof(T6)), + }); + + ICollection IDictionary.Keys => new KeyList(this); + ICollection IDictionary.Values => new ValueList(this); + + IEnumerable IReadOnlyDictionary.Keys => Keys; + IEnumerable IReadOnlyDictionary.Values => Values; + + bool ICollection.IsSynchronized => false; + object ICollection.SyncRoot => this; + + public object? this[string colName] { + get { + if (colName == "id") { + return Id; + } + if (colName == Item1.Name) { + return Item1.Value; + } + if (colName == Item2.Name) { + return Item2.Value; + } + if (colName == Item3.Name) { + return Item3.Value; + } + if (colName == Item4.Name) { + return Item4.Value; + } + if (colName == Item5.Name) { + return Item5.Value; + } + if (colName == Item6.Name) { + return Item6.Value; + } + throw new KeyNotFoundException(); + } + set { + if (colName == "id") { + Id = ExceptionAssert.Cast(value).To(); + } + if (colName == Item1.Name) { + Item1 = (Item1.Name, ExceptionAssert.Cast(value).To()); + } + if (colName == Item2.Name) { + Item2 = (Item2.Name, ExceptionAssert.Cast(value).To()); + } + if (colName == Item3.Name) { + Item3 = (Item3.Name, ExceptionAssert.Cast(value).To()); + } + if (colName == Item4.Name) { + Item4 = (Item4.Name, ExceptionAssert.Cast(value).To()); + } + if (colName == Item5.Name) { + Item5 = (Item5.Name, ExceptionAssert.Cast(value).To()); + } + if (colName == Item6.Name) { + Item6 = (Item6.Name, ExceptionAssert.Cast(value).To()); + } + throw new KeyNotFoundException(); + } + } + + public object? this[int index] => index switch { + 0 => Id, + 1 => Item1.Value, + 2 => Item2.Value, + 3 => Item3.Value, + 4 => Item4.Value, + 5 => Item5.Value, + 6 => Item6.Value, + _ => throw new IndexOutOfRangeException() + }; + + object? IDictionary.this[object key] { + get => this[ExceptionAssert.Cast(key).To()]; + set => this[ExceptionAssert.Cast(key).To()] = value; + } + + public void CopyTo(Array array, int index) { + array.SetValue(Id, index++); + array.SetValue(Item1.Value, index++); + array.SetValue(Item2.Value, index++); + array.SetValue(Item3.Value, index++); + array.SetValue(Item4.Value, index++); + array.SetValue(Item5.Value, index++); + array.SetValue(Item6.Value, index++); + } + + public bool Contains(object key) => + key is string keystr && ContainsKey(keystr); + + public bool ContainsKey(string key) => + Keys.Contains(key); + + public void SetId(int id) => + Id = id; + + public void SetValue1(T1 value) => + Item1 = (Item1.Name, value); + + public void SetValue1(string colName, T1 value) => + Item1 = (colName, value); + + public void SetValue2(T2 value) => + Item2 = (Item2.Name, value); + + public void SetValue2(string colName, T2 value) => + Item2 = (colName, value); + + public void SetValue3(T3 value) => + Item3 = (Item3.Name, value); + + public void SetValue3(string colName, T3 value) => + Item3 = (colName, value); + + public void SetValue4(T4 value) => + Item4 = (Item4.Name, value); + + public void SetValue4(string colName, T4 value) => + Item4 = (colName, value); + + public void SetValue5(T5 value) => + Item5 = (Item5.Name, value); + + public void SetValue5(string colName, T5 value) => + Item5 = (colName, value); + + public void SetValue6(T6 value) => + Item6 = (Item6.Name, value); + + public void SetValue6(string colName, T6 value) => + Item6 = (colName, value); + + public bool TryGetValue(string key, [MaybeNullWhen(false)] out object? value) { + if (!Keys.TryFindIndex(key, out int index)) { + value = null; + return false; + } + value = this[index]; + return true; + } + + void IDictionary.Add(object key, object? value) => + throw new NotSupportedException(); + + void IDictionary.Clear() => + throw new NotSupportedException(); + + void IDictionary.Remove(object key) => + throw new NotSupportedException(); + + IDictionaryEnumerator IDictionary.GetEnumerator() => + throw new NotImplementedException(); + + IEnumerator IEnumerable.GetEnumerator() => + throw new NotImplementedException(); + + public IEnumerator> GetEnumerator() { + throw new NotImplementedException(); + } + + class KeyList : ICollection, IReadOnlyList { + readonly BlueRowStore store; + + public KeyList(BlueRowStore store) => + this.store = store; + + public int Count => 5; + bool ICollection.IsSynchronized => false; + object ICollection.SyncRoot => this; + + public string this[int index] => + index switch { + 0 => "id", + 1 => store.Item1.Name, + 2 => store.Item2.Name, + 3 => store.Item3.Name, + 4 => store.Item4.Name, + 5 => store.Item5.Name, + 6 => store.Item6.Name, + _ => throw new IndexOutOfRangeException() + }; + + public void CopyTo(Array array, int index) { + array.SetValue(store.Id, index++); + array.SetValue(store.Item1.Value, index++); + array.SetValue(store.Item2.Value, index++); + array.SetValue(store.Item3.Value, index++); + array.SetValue(store.Item4.Value, index++); + array.SetValue(store.Item5.Value, index++); + array.SetValue(store.Item6.Value, index++); + } + + public IEnumerator GetEnumerator() { + yield return "id"; + yield return store.Item1.Name; + yield return store.Item2.Name; + yield return store.Item3.Name; + yield return store.Item4.Name; + yield return store.Item5.Name; + yield return store.Item6.Name; + } + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); + } + + class ValueList : ICollection, IReadOnlyList { + readonly BlueRowStore store; + + public ValueList(BlueRowStore store) => + this.store = store; + + public object? this[int index] => + index switch { + 0 => store.Id, + 1 => store.Item1.Value, + 2 => store.Item2.Value, + 3 => store.Item3.Value, + 4 => store.Item4.Value, + 5 => store.Item5.Value, + 6 => store.Item6.Value, + _ => throw new IndexOutOfRangeException() + }; + + public int Count => 5; + public bool IsSynchronized => false; + public object SyncRoot => this; + + public void CopyTo(Array array, int index) { + array.SetValue(store.Id, index++); + array.SetValue(store.Item1.Value, index++); + array.SetValue(store.Item2.Value, index++); + array.SetValue(store.Item3.Value, index++); + array.SetValue(store.Item4.Value, index++); + array.SetValue(store.Item5.Value, index++); + array.SetValue(store.Item6.Value, index++); + } + + public IEnumerator GetEnumerator() { + yield return store.Id; + yield return store.Item1.Value; + yield return store.Item2.Value; + yield return store.Item3.Value; + yield return store.Item4.Value; + yield return store.Item5.Value; + yield return store.Item6.Value; + } + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/ITimeSeriesRow.cs b/src/DataStreamSuite/DataStreamCore/BlueData/ITimeSeriesRow.cs new file mode 100644 index 000000000..5a27bb3cf --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/ITimeSeriesRow.cs @@ -0,0 +1,12 @@ +namespace Bts.DataStreamCore; + +using BCG; + +using mbts.utils; + +using System; + +public interface ITimeSeriesRow : IBlueRow { + int Id => ExceptionAssert.of_type(this["id"]); + DateTime Timestamp { get; } +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/CompositeDataStreamFactory.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/CompositeDataStreamFactory.cs new file mode 100644 index 000000000..7de5bc194 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/CompositeDataStreamFactory.cs @@ -0,0 +1,27 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using System.Collections.Generic; +using System.Linq; + +public class CompositeDataStreamFactory : IDataStreamFactory { + readonly List streamFactories; + + public CompositeDataStreamFactory(IEnumerable streamFactories) { + this.streamFactories = streamFactories.ToList(); + foreach (var rsf in this.streamFactories.OfType()) { + rsf.OnCompositeFactory(this); + } + } + + public IEnumerable GetSupportedStreams() => + streamFactories.SelectMany(f => f.GetSupportedStreams()); + + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) => + streamFactories.Where(f => f.CanCreate(streamId)) + .Select(f => f.Create(streamId, args)) + .ExceptNull() + .FirstOrDefault(); +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamFactory.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamFactory.cs new file mode 100644 index 000000000..d00214014 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamFactory.cs @@ -0,0 +1,66 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using Microsoft.Extensions.Logging; + +using System; +using System.Collections.Generic; +using System.Linq; + +public class DataStreamFactory : IDataStreamFactory { + readonly List factories = new(); + + public static IDataStreamFactory Create(IEnumerable streamFactories) { + var dsf = new DataStreamFactory(); + dsf.factories.AddRange(streamFactories); + return dsf.factories.Count == 1 ? dsf.factories[0] : dsf; + } + + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) => + factories.Where(sf => sf.GetSupportedStreams().Any(m => m.IsMatch(streamId))) + .Select(sf => sf.Create(streamId, args)) + .ExceptNull() + .FirstOrDefault(); + + public IEnumerable GetSupportedStreams() => + factories.SelectMany(sf => sf.GetSupportedStreams()); + + public void Add(IDataStreamFactory streamFactory) => + factories.Add(streamFactory); + + class EmptyDataStreamFactory : IDataStreamFactory { + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) => + null; + + public IEnumerable GetSupportedStreams() => + Enumerable.Empty(); + } +} + +public static class DataStreamFactoryExtensions { + public static void Add(this DataStreamFactory streamFactory, IDataStreamMetadata metadata, Func singleStreamfactory, ILogger? logger = null) { + var sf = new SingleDataStreamFactory(metadata, singleStreamfactory, logger); + streamFactory.Add(sf); + } + + public static void Add(this DataStreamFactory streamFactory, string name, Func singleStreamfactory, ILogger? logger = null) { + var sf = new SingleDataStreamFactory(new DataStreamMetadata() { Name = name }, singleStreamfactory, logger); + streamFactory.Add(sf); + } + + public static void Add(this DataStreamFactory streamFactory, string name, Func, ViewStream?> singleStreamfactory) { + var sf = new SingleDataStreamFactory(new DataStreamMetadata() { Name = name }, singleStreamfactory); + streamFactory.Add(sf); + } + + public static void Add(this DataStreamFactory streamFactory, string[] names, Func singleStreamfactory, ILogger? logger = null) { + if (names.Length == 0) { + throw new ArgumentException("StreamFactory must have at least one name", nameof(names)); + } + var meta = new DataStreamMetadata() { Name = names[0], AlternateNames = names.Skip(1).ToList() }; + var sf = new SingleDataStreamFactory(meta, singleStreamfactory, logger); + streamFactory.Add(sf); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamMetadata.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamMetadata.cs new file mode 100644 index 000000000..53d57791e --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamMetadata.cs @@ -0,0 +1,54 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using bts; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +public interface IDataStreamMetadata { + public string Name { get; } + public IEnumerable AlternateNames { get; } +} + +public class DataStreamMetadata : IDataStreamMetadata { + public string? Name { get; set; } + public List? AlternateNames { get; set; } + + string IDataStreamMetadata.Name => Name ?? string.Empty; + IEnumerable IDataStreamMetadata.AlternateNames => + AlternateNames ?? new(0); + + public static DataStreamMetadata FromType(Type type) { + var friendlyName = type.GetCustomAttribute()?.friendlyName; + var rv = new DataStreamMetadata() { + Name = friendlyName ?? type.Name, + AlternateNames = new(2) { + Log.notnull(type.FullName) + } + }; + if (friendlyName is not null) { + rv.AlternateNames.Add(type.Name); + } + return rv; + } +} + +public static class DataStreamMetadataExtensions { + public static bool IsMatch(this IDataStreamMetadata dsm, string name, bool checkAlternateNames = true, bool ignoreCase = true, bool ignoreUnderscores = true) { + var strComp = ignoreCase ? StringComparison.CurrentCultureIgnoreCase : StringComparison.CurrentCulture; + var nameWithoutUnderscores = new Lazy(() => RemoveUnderscores(name)); + return IsNameMatch(dsm.Name) || (checkAlternateNames && dsm.AlternateNames.Any(IsNameMatch)); + + static string RemoveUnderscores(string s) => + s.Replace("_", ""); + + bool IsNameMatch(string otherName) => + string.Equals(name, otherName, strComp) + || (ignoreUnderscores && string.Equals(nameWithoutUnderscores.Value, RemoveUnderscores(otherName), strComp)); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/IDataStreamFactory.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/IDataStreamFactory.cs new file mode 100644 index 000000000..621455bdf --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/IDataStreamFactory.cs @@ -0,0 +1,25 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using System.Collections.Generic; +using System.Linq; + +public interface IDataStreamFactory { + IEnumerable GetSupportedStreams(); + + ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args); + + bool CanCreate(string streamId) => + GetSupportedStreams().Any(s => s.IsMatch(streamId)); +} + +/// +/// This interface provides a way for IDataStreamFactory's that create ViewStreams +/// that depend specifically on the composite IDataStreamFactory. This was designed +/// specifically for the NLStreamDataFactory. +/// +public interface IRecursiveDataStreamFactory : IDataStreamFactory { + void OnCompositeFactory(IDataStreamFactory compositeFactory); +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/ISingleDataStreamFactory.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/ISingleDataStreamFactory.cs new file mode 100644 index 000000000..23fe386c6 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/ISingleDataStreamFactory.cs @@ -0,0 +1,18 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using System.Collections.Generic; + +public interface ISingleStreamFactory : IDataStreamFactory { + IDataStreamMetadata Metadata { get; } + + ViewStream? Create(IEnumerable<(string? Name, int? Index, object? Value)> args); + + ViewStream? IDataStreamFactory.Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) => + Metadata.IsMatch(streamId) ? Create(args) : null; + + IEnumerable IDataStreamFactory.GetSupportedStreams() => + new[] { Metadata }; +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/IStreamArgumentParser.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/IStreamArgumentParser.cs new file mode 100644 index 000000000..52f43e046 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/IStreamArgumentParser.cs @@ -0,0 +1,6 @@ +#nullable enable +namespace Bts.DataStreamCore; + +public interface IStreamArgumentParser { + bool TryParse(string? name, string? valueStr, out object? result); +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs index a889290cb..d809f42cf 100644 --- a/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs @@ -8,33 +8,36 @@ namespace BCG { using System.Collections; using System.Reflection; using System.IO; + using System.Collections.Generic; + using System.Linq; + using bts.collections; + using System.Diagnostics.CodeAnalysis; + using Spark.Core.Collections; using Bts.DataStreamCore; + using Microsoft.Extensions.Options; - public class PrimitiveManager { - static MarketDataStreamFactory mdStreamFactory; - private IDictionary myTypeLookupTable = new Hashtable(); + public class PrimitiveManager : IDataStreamFactory { private static PrimitiveManager mySelf; - private ArrayList myAssemblyNames = new ArrayList(); + readonly List myAssemblyNames = new(); + readonly Dictionary myTypeLookupTable = new(); - private PrimitiveManager() { - myLoadAllTypes(); - } + public PrimitiveManager(IOptions config) : this(config.Value) { } - private void myLoadAllTypes() { + public PrimitiveManager(PrimitiveManagerConfig config) { AppDomain currentDomain = AppDomain.CurrentDomain; currentDomain.AssemblyResolve += new ResolveEventHandler(myAssemblyResolver); loadAssembly(this.GetType().Assembly.FullName); - var assemblies = new string[] { "DataStream.dll", "SharedStreams.dll" }; - var config_assemblies = Configuration.get().Assemblies; - bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes config_assemblies: {config_assemblies}"); - foreach (string assemblyName in assemblies/*config_assemblies*/) { + foreach (string assemblyName in config.Assemblies.empty_if_null()) { bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes assemblyName: {assemblyName}"); string resolvedAssemblyPath = assemblyName; if (!System.IO.Path.IsPathRooted(assemblyName)) { resolvedAssemblyPath = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assemblyName)); } - loadAssembly(resolvedAssemblyPath); + loadAssembly(resolvedAssemblyPath, IncludeType); } + + bool IncludeType(Type t) => + !config.ExcludeTypes.empty_if_null().Any(tname => tname == t.Name); } private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) { @@ -48,12 +51,7 @@ namespace BCG { /// Loads all the Stream types associated with the assembly /// /// - public void loadAssembly(string assemblyName) { - Func is_blue_orb = (t) => { - //if (!assemblyName.Contains("SharedStreams.dll")) - // return true; - return t.Namespace == "bts.BlueOrb";// || t.Namespace == "BCG"; - }; + public void loadAssembly(string assemblyName, Func typeFilter = null) { bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}"); if (myAssemblyNames.Contains(assemblyName)) return; @@ -62,8 +60,12 @@ namespace BCG { GetType().Assembly : Assembly.LoadFrom(assemblyName); - foreach (Type type in loadedAssembly.GetTypes()) - if (!type.IsInterface && is_blue_orb(type) + IEnumerable types = loadedAssembly.GetTypes(); + if (typeFilter is not null) { + types = types.Where(typeFilter); + } + foreach (Type type in types) + if (!type.IsInterface && !type.IsAbstract && type.IsSubclassOf(typeof(ViewStream)) && type.IsPublic) { @@ -79,7 +81,7 @@ namespace BCG { break; } if (isVisible) { - // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName); + Log.assert_d(!myTypeLookupTable.ContainsKey(userRep)); myTypeLookupTable[userRep] = type; } } @@ -95,26 +97,28 @@ namespace BCG { return retVal; } + public IEnumerable GetSupportedTypes() => + myTypeLookupTable.Values.Cast(); + public IEnumerable GetSupportedStreams() => + GenericDictionaryWrapper.Create(myTypeLookupTable).Select(kv => new DataStreamMetadata() { + Name = kv.Key, + AlternateNames = new() { kv.Value.Name, kv.Value.FullName } + }); - public static PrimitiveManager instance { - get { return mySelf = mySelf == null ? new PrimitiveManager() : mySelf; } - } + public Type ResolveType(string streamId) => + TryResolveType(streamId, out var t) ? t + : throw new ArgumentException($"Failed to resolve ViewStream type '{streamId}'", nameof(streamId)); - public static void SetMarketDataClient(object value) { - Log.isnull(mdStreamFactory); - mdStreamFactory = new(value); - } - - /// - /// Returns the Type name corresponding to the primitive name - /// - /// - /// - public string find(string friendlyName) { - if (myTypeLookupTable[friendlyName] == null) - throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types"); - return (myTypeLookupTable[friendlyName] as Type).Name; + public bool TryResolveType(string streamId, [NotNullWhen(true)] out Type result) { + result = GetSupportedStreams() + .Where(s => s.IsMatch(streamId)) + .Select(s => s.Name) + .Prepend(streamId) + .Select(myTypeLookupTable.GetValueOrDefault) + .ExceptNull() + .FirstOrDefault(); + return result is not null; } /// @@ -130,19 +134,110 @@ namespace BCG { constructorArgs = new object[args.Count]; args.CopyTo(constructorArgs, 0); } - var type = myTypeLookupTable[friendlyName] as Type - ?? throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types"); - if (type.HasCustomAttribute()) { - if (mdStreamFactory is null) { - Log.Gen.error("Live md not available for streams"); - return null; - } - return mdStreamFactory.Create(type, constructorArgs); + if (myTypeLookupTable[friendlyName] == null) + throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types"); + try { + return + constructorArgs == null ? + Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type) as ViewStream : + Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type, constructorArgs) as ViewStream; + } catch (Exception ex) { + Log.Gen.warn($"PrimitiveManager failed to construct ViewStream type '{friendlyName}': {ex}"); + } + return null; + } + + public ViewStream Create(string streamId, IEnumerable<(string Name, int? Index, object Value)> args) => + TryResolveType(streamId, out var type) ? Create(type, args) : null; + + /// + /// Constructs and returns a ViewStream of type corresponding to the friendly name + /// + /// friendly primitive name + /// args can be null or an empty ICollection if the paramterless constructor + /// is the desired one + /// + public ViewStream Create(Type type, IEnumerable<(string Name, int? Index, object Value)> args) { + var ctrs = type.GetConstructors(BindingFlags.Instance | BindingFlags.Public); + var argList = args.ToList(); + const int MAX_ALLOWED_ARGS = 64; + if (argList.Count > MAX_ALLOWED_ARGS) { + throw new ArgumentException( + $"ViewStream arg count ({argList.Count}) exceeds max allowed ({MAX_ALLOWED_ARGS})", nameof(args)); + } + if (argList.Any(a => !(a.Name?.Length > 0) && !a.Index.HasValue)) { + throw new ArgumentException( + "One or more ViewStream args lack both Name and Index (unable to assign a constructor param)", nameof(args)); + } + var indexedArgs = argList.Where(a => a.Index.HasValue).ToList(); + var namedArgs = argList.Where(a => a.Name?.Length > 0).ToList(); + if (!namedArgs.Select(a => a.Name).IsDistinct()) { + throw new ArgumentException("ViewStream arg names are not unique"); + } + if (!indexedArgs.Select(a => a.Index.Value).IsDistinct()) { + throw new ArgumentException("ViewStream arg indices are not unique"); + } + // Constructor params with default values may be omitted from the input 'args'. Therefore, + // 'minArgCount' >= 'argList.Count'. + var minArgCount = Math.Max(indexedArgs.Select(a => a.Index.Value).MaxOrDefault(), argList.Count); + foreach (var ctr in ctrs) { + if (TryAssignConstructorArgs(ctr, minArgCount, argList, indexedArgs, namedArgs, out var ctrArgs)) { + return ctr.Invoke(ctrArgs) as ViewStream; + } + } + return null; + + static bool TryAssignConstructorArgs( + ConstructorInfo ctr, + int minArgCount, + List<(string Name, int? Index, object Value)> argList, + List<(string Name, int? Index, object Value)> indexedArgs, + List<(string Name, int? Index, object Value)> namedArgs, + [NotNullWhen(true)] out object[] ctrArgs + ) { + ctrArgs = null; + var ctrParams = ctr.GetParameters(); + if (ctrParams.Length < minArgCount) { + return false; + } + var ctrArgList = new List(); + int remInputArgs = argList.Count; + for (int i = 0; i < ctrParams.Length; i++) { + var ctrParam = ctrParams[i]; + var paramType = ctrParams[i].ParameterType; + if (indexedArgs.TryGetFirst(a => CanAssignInputArg(ctrParam, a), out var a)) { + remInputArgs--; + ctrArgList.Add(a.Value); + } else if (ctrParam.HasDefaultValue) { + ctrArgList.Add(ctrParam.DefaultValue); + } else { + return false; + } + } + if (remInputArgs > 0) { + return false; + } + Log.assert(ctrArgList.Count == ctrParams.Length); + ctrArgs = ctrArgList.ToArray(); + return true; + } + + static bool CanAssignInputArg(ParameterInfo pi, (string Name, int? Index, object Value) arg) { + if (!string.IsNullOrEmpty(arg.Name) && arg.Name != pi.Name) { + return false; + } + if (arg.Index.HasValue && arg.Index.Value != pi.Position) { + return false; + } + if (arg.Value is null) { + if (!pi.ParameterType.IsValueType && !pi.ParameterType.IsNullable()) { + return false; + } + } else if (!pi.ParameterType.IsAssignableFrom(arg.Value.GetType())) { + return false; + } + return true; } - return - constructorArgs == null ? - Activator.CreateInstance(type) as ViewStream : - Activator.CreateInstance(type, constructorArgs) as ViewStream; } /// diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManagerConfig.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManagerConfig.cs new file mode 100644 index 000000000..881433190 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManagerConfig.cs @@ -0,0 +1,9 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using System.Collections.Generic; + +public class PrimitiveManagerConfig { + public List? Assemblies { get; set; } + public List? ExcludeTypes { get; set; } +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/SingleDataStreamFactory.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/SingleDataStreamFactory.cs new file mode 100644 index 000000000..d0d87d82b --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/SingleDataStreamFactory.cs @@ -0,0 +1,59 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using Microsoft.Extensions.Logging; + +using System; +using System.Collections.Generic; + +class SingleDataStreamFactory : ISingleStreamFactory { + readonly Func, ViewStream?> factory; + + public SingleDataStreamFactory(IDataStreamMetadata metadata, Func, ViewStream?> factory) { + Metadata = metadata; + this.factory = factory; + } + + public IDataStreamMetadata Metadata { get; } + + public ViewStream? Create(IEnumerable<(string? Name, int? Index, object? Value)> args) => + factory(args); + + public IEnumerable GetSupportedStreams() => + new[] { Metadata }; +} + +class SingleDataStreamFactory : ISingleStreamFactory { + readonly Func factory; + readonly ILogger? logger; + readonly StreamFactoryArgStore.TryGetParamsDelegate tryGetParams; + + public SingleDataStreamFactory(IDataStreamMetadata metadata, Func factory) + : this(metadata, factory, null) { } + + public SingleDataStreamFactory(IDataStreamMetadata metadata, Func factory, ILogger? logger, StreamFactoryArgStore.TryGetParamsDelegate? tryGetParams = null) { + Metadata = metadata; + this.factory = factory; + this.logger = logger; + this.tryGetParams = tryGetParams ?? new(StreamFactoryArgStore.TryGetParams); + } + + public IDataStreamMetadata Metadata { get; } + + public ViewStream? Create(IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (!tryGetParams(args, out TParams? streamParams, out var err)) { + logger?.LogInvalidParams(Metadata.Name, err); + return null; + } + return Create(streamParams); + } + + public ViewStream? Create(TParams streamParams) { + return factory(streamParams); + } + + public IEnumerable GetSupportedStreams() => + new[] { Metadata }; +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgumentAttribute.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgumentAttribute.cs new file mode 100644 index 000000000..e5db96827 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamArgumentAttribute.cs @@ -0,0 +1,19 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using System; + +[AttributeUsage(AttributeTargets.Property)] +public class StreamArgumentAttribute : Attribute { + public StreamArgumentAttribute(params string[] names) { + Names = names; + RegexNames = Array.Empty(); + } + + public string[] Names { get; } + public string[] RegexNames { get; set; } + public int Position { get; set; } = -1; + public bool IsRequired { get; set; } + public bool AllowCommaSeparatedAliases { get; set; } = true; + public Type? ParserType { get; set; } +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamFactoryArgStore.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamFactoryArgStore.cs new file mode 100644 index 000000000..b02da2d52 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamFactoryArgStore.cs @@ -0,0 +1,329 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using mbts.utils; + +using Microsoft.Extensions.Logging; + +using Spark.Core; + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Text.RegularExpressions; + +public class StreamFactoryArgStore { + readonly ILogger? logger; + readonly string streamType; + ImmutableList<(string? Name, int? Index, object? Value)> args; + + public StreamFactoryArgStore(IEnumerable<(string? Name, int? Index, object? Value)> args, string? streamType = null, ILogger? logger = null) { + this.args = args.ToImmutableList(); + this.streamType = streamType ?? "anonStream"; + this.logger = logger; + } + + public delegate bool TryGetParamsDelegate(IEnumerable<(string? Name, int? Index, object? Value)> args, [NotNullWhen(true)]out T? result, [NotNullWhen(false)] out string? errorMessage); + public delegate bool TryConvert(object? obj, [NotNullWhen(true)]out T? result); + + public static bool TryGetParams( + IEnumerable<(string? Name, int? Index, object? Value)> args, + [NotNullWhen(true)] out T? result, + [NotNullWhen(false)] out string? errorMessage + ) => + new StreamFactoryArgStore(args).TryGetParams(out result, out errorMessage); + + public bool TryGetParams([MaybeNull][NotNullWhen(true)] out T result, [NotNullWhen(false)] out string? errorMessage) { + var props = typeof(T).GetProperties(); + var res = Activator.CreateInstance(); + if (res is null) { + errorMessage = $"Failed to instantiate {typeof(T)}"; + result = default; + return false; + } + foreach (var p in props) { + var att = p.GetCustomAttribute(); + IEnumerable names = att?.Names ?? new[] { p.Name }; + if (att?.AllowCommaSeparatedAliases ?? false) { + names = names.SelectMany(n => n.Split(',')); + } + IEnumerable regexNames = att?.RegexNames ?? Enumerable.Empty(); + if (TryGet(names, out var arg) || TryGetRegex(regexNames, out arg)) { + SetValue(arg); + } else if (att is not null && att.Position >= 0 && args.Skip(att.Position).TryGetFirst(out arg)) { + SetValue(arg); + } else if (att?.IsRequired ?? false) { + errorMessage = $"Missing required param '{p.Name}'"; + result = default; + return false; + } + + void SetValue((string? Name, int? Index, object? Value) arg) { + var (name, index, val) = arg; + if (val is string valStr && att?.ParserType is Type parserType) { + var parser = ExceptionAssert.of_type(Activator.CreateInstance(parserType)); + if (!parser.TryParse(name, valStr, out val)) { + throw new InvalidOperationException($"Failed to parse arg with custom parser of type {parserType}"); + } + } + var pType = p.PropertyType; + bool isNullable = pType.IsNullable(); + if (isNullable) { + pType = pType.GetGenericArguments().Single(); + } + var vType = val?.GetType() ?? typeof(object); + if (pType == typeof(DateTime)) { + SetValueType(BlueDataUtility.GetDateTime(val)); + } else if (pType == typeof(TimeSpan)) { + SetValueType(BlueDataUtility.GetTimeSpan(val)); + } else if (pType == typeof(string)) { + p.SetValue(res, val as string ?? val?.ToString()); + } else if (pType.IsEnum) { + p.SetValue(res, BlueDataUtility.GetEnum(pType, val)); + } else if (vType.IsAssignableTo(pType) || (val is null && (!pType.IsValueType || isNullable))) { + p.SetValue(res, val); + } else { + p.SetValue(res, Convert.ChangeType(val, pType)); + } + + void SetValueType(TProp? pVal) where TProp : struct { + if (isNullable) { + p.SetValue(res, pVal); + } else { + p.SetValue(res, pVal.assert_value()); + } + } + } + } + errorMessage = null; + result = res; + return true; + } + + public bool GetInt32(string name, [NotNullWhen(true)] out int? result) => + GetInt32(out result, true, name); + + public bool GetInt32([NotNullWhen(true)] out int? result, bool required, params string[] names) { + result = GetInt32(names); + if (required && !result.HasValue) { + logger.LogMissingParam(streamType, names.Length switch { + 0 => "Int32", + _ => names[0], + }); + } + return result.HasValue; + } + + public int? GetInt32(params string[] names) => + GetInt32(true, names); + + public int? GetInt32(bool required, params string[] names) { + if (names.Length == 0) { + return null; + } + var obj = Get(names); + if (obj is int i) { + return i; + } + var s = obj as string ?? obj?.ToString(); + if (int.TryParse(s, out i)) { + return i; + } + if (required && logger is not null) { + logger.LogMissingParam(streamType, names[0]); + } + return null; + } + + public bool GetTimeSpan(string name, out TimeSpan? result) { + result = GetTimeSpan(true, new[] { name }); + return result.HasValue; + } + + public TimeSpan? GetTimeSpan(string name) => + GetTimeSpan(true, new[] { name }); + + public TimeSpan? GetTimeSpan(bool required, string[] preferredNames) => + preferredNames + .Select(n => BlueDataUtility.GetTimeSpan(Get(n))) + .Where(ts => ts.HasValue) + .FirstOrDefault(); + + public IEnumerable GetStrings() => + args.Select(a => a.Value).OfType(); + + // First, parses the args associated with preferredNames and returns the first successful + // parse result. If none were successful, tries to parse every arg. + public T? GetEnum(params string[] preferredNames) where T : struct { + var candidateObjs = preferredNames.Select(n => Get(n)).Concat(args.Select(a => a.Value)); + foreach (var obj in candidateObjs) { + if (obj is T res) { + return res; + } + if (obj is string s && Enum.TryParse(s, true, out res)) { + return res; + } + } + return null; + } + + public IEnumerable GetDateTimes() => + args.Select(a => convertToDateTime(a.Value)).SelectWhereValues(); + + public bool GetDateTime(string name, [NotNullWhen(true)] out DateTime? result) { + result = GetDateTime(true, name); + return result.HasValue; + } + + public bool GetDateTime([NotNullWhen(true)] out DateTime? result, params string[] names) { + result = GetDateTime(names); + return result.HasValue; + } + + public bool GetDateTime([NotNullWhen(true)] out DateTime? result, out int index, bool removeArg) { + int i = 0; + foreach (var arg in args) { + var dt = convertToDateTime(arg.Value); + if (dt.HasValue) { + result = dt.Value; + index = i; + if (removeArg) { + args = args.RemoveAt(i); + } + return true; + } + } + result = null; + index = -1; + return false; + } + + public DateTime? GetDateTime(params string[] names) => + GetDateTime(true, names); + + public DateTime? GetDateTime(bool required, params string[] names) => + names.Select(n => Get(n)).Select(convertToDateTime).Where(dt => dt.HasValue).FirstOrDefault(); + + public object? Get(params string[] names) => + Get(true, names); + + public object? Get(out int index, params string[] names) => + Get(true, out index, names); + + public object? Get(bool required, params string[] names) => + Get(required, out _, names); + + public object? Get(bool required, out int index, params string[] names) { + if (names.Length == 0) { + index = -1; + return null; + } + if (!TryGet(names, out var result, out index) && required && logger is not null) { + logger.LogMissingParam(streamType, names[0]); + } + return result.Value; + } + + public bool TryGetRegex(string name, out (string? Name, int? Index, object? Value) result) => + TryGet(new[] { name }, out result, checkDefaultMatch: false, checkRegexMatch: true); + + public bool TryGetRegex(IEnumerable names, out (string? Name, int? Index, object? Value) result) => + TryGet(names, out result, checkDefaultMatch: false, checkRegexMatch: true); + + public bool TryGet(IEnumerable names, out (string? Name, int? Index, object? Value) result, bool checkDefaultMatch = true, bool checkRegexMatch = false) => + TryGet(names, out result, out _, checkDefaultMatch, checkRegexMatch); + + public bool TryGet(IEnumerable names, out (string? Name, int? Index, object? Value) result, out int index, bool checkDefaultMatch = true, bool checkRegexMatch = false) { + if (checkDefaultMatch) { + foreach (var name in names.SelectMany(Variants)) { + if (args.TryGetFirst(a => a.Name == name, out var a, out index)) { + result = a; + return true; + } + } + } + if (checkRegexMatch) { + foreach (var name in names) { + if (args.TryGetFirst(a => Regex.IsMatch(a.Name ?? string.Empty, name, RegexOptions.IgnoreCase), out var a, out index)) { + result = a; + return true; + } + } + } + index = -1; + result = default; + return false; + + IEnumerable Variants(string s) { + if (string.IsNullOrEmpty(s)) { + yield break; + } + yield return s; + // Convert to snake_case + var bd = new StringBuilder(char.ToLower(s[0])); + for (int i = 1; i < s.Length; i++) { + if (i > 1 && char.IsUpper(s[i]) && char.IsLower(s[i - 1])) { + bd.Append('_'); + } + bd.Append(char.ToLower(s[i])); + } + yield return bd.ToString(); + // Convert to camelCase + bd.Clear(); + for (int i = 0; i < s.Length; i++) { + int numUnderscores = 0; + while (s[i + numUnderscores] == '_' & (i + numUnderscores < s.Length)) { + numUnderscores++; + } + if (numUnderscores > 0) { + i += numUnderscores; + if (i >= s.Length) { + break; + } + bd.Append(char.ToUpper(s[i])); + } else { + bd.Append(s[i]); + } + } + yield return bd.ToString(); + } + } + + static DateTime? convertToDateTime(object? obj) => + tryConvert(obj, out DateTime dt) ? dt : null; + + static bool tryConvert(object? obj, out DateTime res) { + if (obj is DateTime dt) { + res = dt; + return true; + } + if (obj is string s) { + // if (OffsetDateTimePattern) + if (DateTimeOffset.TryParseExact(s, "yyyy-MM-ddTHH:mm:ssK", null, System.Globalization.DateTimeStyles.None, out var dto)) { + res = dto.ToUniversalTime().DateTime; + return true; + } + if (DateTimeOffset.TryParseExact(s, "yyyy-MM-ddTHH:mm:ss.fffffffffK", null, System.Globalization.DateTimeStyles.None, out dto)) { + res = dto.ToUniversalTime().DateTime; + return true; + } + if (DateTime.TryParse(s, null, System.Globalization.DateTimeStyles.RoundtripKind, out res)) { + return true; + } + if (DateTimeOffset.TryParse(s, out var resOff)) { + res = resOff.ToUniversalTime().DateTime; + return true; + } + return DateTime.TryParse(s, out res); + } + res = default; + return false; + } + +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamFactoryLogs.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamFactoryLogs.cs new file mode 100644 index 000000000..be1c8dadd --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/StreamFactoryLogs.cs @@ -0,0 +1,11 @@ +namespace Bts.DataStreamCore; + +using Microsoft.Extensions.Logging; + +public static partial class StreamFactoryLogs { + [LoggerMessage(EventId = 1, Level = LogLevel.Warning, Message = "Unable to create '{StreamType}' stream. Missing required parameter '{Parameter}'.")] + public static partial void LogMissingParam(this ILogger logger, string streamType, string parameter); + + [LoggerMessage(EventId = 2, Level = LogLevel.Warning, Message = "Unable to create '{StreamType}' stream. {error}.")] + public static partial void LogInvalidParams(this ILogger logger, string streamType, string error); +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/WellKnownStreamArgumentType.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/WellKnownStreamArgumentType.cs new file mode 100644 index 000000000..c9e586310 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/WellKnownStreamArgumentType.cs @@ -0,0 +1,58 @@ +#nullable enable +namespace Bts.DataStreamCore; + +public static class WellKnownStreamArgumentAliases { + /// + /// Aliases for a Venue argument. + /// + public const string Venue = "venue,exchange,exch"; + + /// + /// Aliases for a Type argument. + /// + public const string Asset = "symbol,sym,product,asset"; + + /// + /// Aliases for a Type argument. + /// + public const string Type = "type,typ"; + + /// + /// Aliases for a SecurityType argument. Some are potentially ambiguous, like 'type' and 'typ'. + /// If this conflicts with other arguments, consider using . + /// + public const string SecurityType = "secType,secTyp,securityType,securityTyp,type,typ"; + + /// + /// Aliases for a SecurityName argument. Some are potentially ambiguous, like 'name'. If this + /// conflicts with other arguments, consider using . + /// + public const string SecurityName = "name,sec,secName,security"; + + public const string QualifiedSecurityType = "secType,secTyp,securityType,securityTyp"; + public const string QualifiedSecurityName = "sec,secName,security"; + + public const string VenueRegex = "^(venue|exchange|exch)$"; + public const string AssetRegex = "^(symbol|sym|asset|product)$"; + public const string TypeRegex = "^(type|typ)$"; + public const string SecurityTypeRegex = "^(secType|secTyp|type|typ)$"; + public const string SecurityNameRegex = "^(sec|security|name|secName|securityName)$"; + + public const string ShowVenueRegex = "^(show|include)(Exch|Venue)$"; + public const string ShowAssetRegex = "^(show|include)(Symbol|Sym|Asset|Product)$"; + public const string ShowSecurityTypeRegex = "^(show|include)(Sec|Security)?(Type|Typ)$"; + public const string ShowSecurityNameRegex = "^(show|include)(Sec|Security|Name|SecName|SecurityName)$"; + public const string ShowExpirationRegex = "^(show|include)(Exp(iration|Date)?)$"; + public const string ShowCashPerPointRegex = "^(show|include)(CashPerPoint|Cpp)$"; + + public const string ShowLegVenueRegex = "^(show|include)Leg(Exch|Venue)$"; + public const string ShowLegSecurityTypeRegex = "^(show|include)Leg(Sec|Security)?(Type|Typ)$"; + public const string ShowLegAssetRegex = "^(show|include)Leg(Symbol|Sym|Asset|Product)$"; + public const string ShowLegExpirationRegex = "^(show|include)Leg(Exp(iration|Date)?)$"; + public const string ShowLegNameRegex = "^(show|include)Leg(Name|Desc)$"; + public const string ShowLegCashPerPointRegex = "^(show|include)Leg(CashPerPoint|Cpp)$"; + + public const string ReplaySpeedRegex = $"^(replay|replaySpeed|speed|rate|velocity)$"; + public const string BeginDateRegex = "(begin|start)(date(Time)?|time)?"; + public const string EndDateRegex = "(end|stop)(date(Time)?|time)?"; +} diff --git a/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs b/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs index fc036feb4..ead934004 100644 --- a/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs +++ b/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs @@ -3,6 +3,7 @@ namespace Bts.DataStreamCore.Services; using BCG; using BCG.Utilities.General; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; public static class DataStreamCoreServices { @@ -14,4 +15,11 @@ public static class DataStreamCoreServices { ViewStreamDatabase.SetGlobalInstance(new ViewStreamDatabase(sp.GetRequiredService())); return ViewStreamDatabase.instance; }); + + public static IServiceCollection AddPrimitiveStreams(this IServiceCollection services, IConfiguration configurationSection) => + services.Configure(configurationSection) + .AddSingleton(); + + public static IServiceCollection AddCompositeDataStreamFactory(this IServiceCollection services) => + services.AddSingleton(); } diff --git a/src/DataStreamSuite/DataStreamExpressions/Assignment.cs b/src/DataStreamSuite/DataStreamExpressions/Assignment.cs index 86590fad7..060b7ab6a 100644 --- a/src/DataStreamSuite/DataStreamExpressions/Assignment.cs +++ b/src/DataStreamSuite/DataStreamExpressions/Assignment.cs @@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions; using BCG; +using Bts.DataStreamCore; + class Assignment : StatementExpression { readonly StatementExpression expression; readonly string id; @@ -19,8 +21,8 @@ class Assignment : StatementExpression { return result; } - public override ViewStream? myGetView(IDictionary env) { - var result = expression.getView(env); + public override ViewStream? myGetView(IDictionary env, IDataStreamFactory streamFactory) { + var result = expression.getView(env, streamFactory); env[id] = expression; return result; } diff --git a/src/DataStreamSuite/DataStreamExpressions/DataStreamExpression.cs b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpression.cs index 7f84a8fad..842ff894d 100644 --- a/src/DataStreamSuite/DataStreamExpressions/DataStreamExpression.cs +++ b/src/DataStreamSuite/DataStreamExpressions/DataStreamExpression.cs @@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions; using BCG; +using Bts.DataStreamCore; + using System.Text; public class DataStreamExpression { @@ -31,10 +33,10 @@ public class DataStreamExpression { return sb.ToString(); } - public ViewStream? getView(IDictionary env) { + public ViewStream? getView(IDictionary env, IDataStreamFactory streamFactory) { for (int i = 0; i < Statements.Count - 1; i++) { - Statements[i].getView(env); + Statements[i].getView(env, streamFactory); } - return Statements[^1].getView(env); + return Statements[^1].getView(env, streamFactory); } } diff --git a/src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs b/src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs index 1b90cd17e..5c0265d22 100644 --- a/src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs +++ b/src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs @@ -2,7 +2,8 @@ namespace Bts.DataStreamExpressions; using BCG; -using System.Collections; +using Bts.DataStreamCore; + using System.Collections.Immutable; public class PrimitiveExpression : StatementExpression { @@ -20,7 +21,7 @@ public class PrimitiveExpression : StatementExpression { } public override string print(int startCol, IList activeBindings) { - string prelude = "new " + PrimitiveManager.instance.find(myName) + " ("; + string prelude = "new " + myName + " ("; return prelude + myPrintArgs(startCol + prelude.Length, activeBindings) + ")"; } @@ -31,21 +32,19 @@ public class PrimitiveExpression : StatementExpression { return prelude + myPrintViewArgs(startCol + prelude.Length) + ")"; } - public override ViewStream myGetView(IDictionary env) { - ArrayList resolvedArgs = new ArrayList(); + public override ViewStream? myGetView(IDictionary env, IDataStreamFactory streamFactory) { + var resolvedArgs = myArgs.Select((arg, i) => ((string?)arg.Name, string.IsNullOrEmpty(arg.Name) ? (int?)i : null, ResolveArg(arg))); + return streamFactory.Create(myName, resolvedArgs); - foreach (var arg in myArgs) { + object? ResolveArg(Argument arg) { if (arg.StatementExp is not null) { - resolvedArgs.Add(arg.StatementExp.myGetView(env)); - continue; + return arg.StatementExp.myGetView(env, streamFactory); } if (arg.QuotedView is not null) { - resolvedArgs.Add(arg.QuotedView.Value); - continue; + return arg.QuotedView.Value; } - resolvedArgs.Add(arg.Obj); + return arg.Obj; } - return PrimitiveManager.instance.build(myName, resolvedArgs); } string myPrintSingleArg(int atColumn, object? o, IList activeBindings) { diff --git a/src/DataStreamSuite/DataStreamExpressions/StatementExpression.cs b/src/DataStreamSuite/DataStreamExpressions/StatementExpression.cs index 211c290bb..7eed8636c 100644 --- a/src/DataStreamSuite/DataStreamExpressions/StatementExpression.cs +++ b/src/DataStreamSuite/DataStreamExpressions/StatementExpression.cs @@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions; using BCG; +using Bts.DataStreamCore; + public abstract class StatementExpression { ViewStream? view; @@ -13,10 +15,10 @@ public abstract class StatementExpression { public virtual string? printView(int startCol) => null; - public ViewStream? getView(IDictionary env) => - view ??= myGetView(env); + public ViewStream? getView(IDictionary env, IDataStreamFactory streamFactory) => + view ??= myGetView(env, streamFactory); - public virtual ViewStream? myGetView(IDictionary env) => + public virtual ViewStream? myGetView(IDictionary env, IDataStreamFactory streamFactory) => null; public static string makeCSString(string s) { diff --git a/src/DataStreamSuite/DataStreamExpressions/ViewNameExpression.cs b/src/DataStreamSuite/DataStreamExpressions/ViewNameExpression.cs index 5d4e3f6e2..a1dea49d0 100644 --- a/src/DataStreamSuite/DataStreamExpressions/ViewNameExpression.cs +++ b/src/DataStreamSuite/DataStreamExpressions/ViewNameExpression.cs @@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions; using BCG; +using Bts.DataStreamCore; + public class ViewNameExpression : StatementExpression { readonly string id; @@ -14,6 +16,6 @@ public class ViewNameExpression : StatementExpression { public override string printView(int startCol) => id; - public override ViewStream? myGetView(IDictionary env) => - env.TryGetValue(id, out var exp) ? exp.getView(env) : new TableStream(id); + public override ViewStream? myGetView(IDictionary env, IDataStreamFactory streamFactory) => + env.TryGetValue(id, out var exp) ? exp.getView(env, streamFactory) : new TableStream(id); } diff --git a/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs index 7467cbd90..3ba431cd1 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs @@ -4,6 +4,7 @@ using BCG; using bts; +using Bts.DataStream; using Bts.DataStreamCore; using Bts.DataStreamExpressions; using Bts.PubSub; @@ -28,13 +29,28 @@ public sealed class DataStreamPublisherService : IHostedService, IDisposable { readonly DataStreamPublisherConfig config; readonly IPubTableBroker pubTableBroker; readonly IWorkScheduler workScheduler; + readonly IDataStreamFactory streamFactory; List? resources = new(); - public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, IOptions config) - : this(pubTableBroker, workScheduler, config.Value) { } + public DataStreamPublisherService( + IPubTableBroker pubTableBroker, + IEnumerable streamFactories, + IWorkScheduler workScheduler, + IOptions config + ) + : this(pubTableBroker, streamFactories, workScheduler, config.Value) { } - public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, DataStreamPublisherConfig config) { + public DataStreamPublisherService( + IPubTableBroker pubTableBroker, + IEnumerable streamFactories, + IWorkScheduler workScheduler, + DataStreamPublisherConfig config + ) { this.pubTableBroker = pubTableBroker; + var streamFactoryList = streamFactories.ToList(); + streamFactory = streamFactoryList.Count == 1 + ? streamFactoryList[0] + : new CompositeDataStreamFactory(streamFactoryList); this.workScheduler = workScheduler; this.config = config; } @@ -48,7 +64,7 @@ public sealed class DataStreamPublisherService : IHostedService, IDisposable { Log.Gen.warn("Skipping PublishedStream because it doesn't have a name (must define 'Name' in config json)."); continue; } - var vs = CreateViewStream(publishedStream); + var vs = CreateViewStream(streamFactory, publishedStream); if (vs is null) { continue; } @@ -72,7 +88,7 @@ public sealed class DataStreamPublisherService : IHostedService, IDisposable { resources?.Clear(); } - static ViewStream? CreateViewStream(PublishedStream ps) { + static ViewStream? CreateViewStream(IDataStreamFactory streamFactory, PublishedStream ps) { Log.Gen.info( $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}"); if (string.IsNullOrEmpty(ps.Name)) { @@ -80,7 +96,7 @@ public sealed class DataStreamPublisherService : IHostedService, IDisposable { $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}"); return null; } - var vs = buildStream(ps.Expression); + var vs = buildStream(streamFactory, ps.Expression); if (vs is null) { Log.Gen.warn( $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}"); @@ -89,14 +105,14 @@ public sealed class DataStreamPublisherService : IHostedService, IDisposable { return vs; } - static ViewStream? buildStream(string? streamExpression) { + static ViewStream? buildStream(IDataStreamFactory streamFactory, string? streamExpression) { if (streamExpression is null) { return null; } try { Log.Gen.info($"Building ViewStream: {streamExpression}"); var exp = DataStreamExpressionsInterpreter.Evaluate(streamExpression); - return exp.getView(new Dictionary(0)); + return exp.getView(new Dictionary(0), streamFactory); } catch (Exception ex) { Log.assert(false, $"Could not build stream. expression: {streamExpression}; ex: {ex}"); return null; diff --git a/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherServices.cs b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherServices.cs index 1c1e9d1a4..8cd3fade4 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherServices.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherServices.cs @@ -19,6 +19,7 @@ public static class DataStreamPublisherServices { .AddPubSubClient(configurationSection.GetSection("PubSub")) .AddPubTableBroker() .AddRemoteQueryManager() + .AddPrimitiveStreams(configurationSection.GetSection("PrimitiveStream")) .Configure(configurationSection.GetSection("DataStreamPublisher")) .AddHostedService(); } diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs index 09ff6d239..bf71a9960 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs @@ -22,8 +22,7 @@ using Bts.DataStreamExpressions; // using BCG.Evaluator.DataStreamExpression; // using BCG.Utilities.Win32; using System.Threading.Tasks; - - +using Bts.DataStreamCore; namespace BCG { @@ -49,9 +48,16 @@ namespace BCG private ViewStream? myAllTagsStream; private string myKQ; private TextWriter? myConsoleOutput; + readonly PrimitiveManager primitiveManager; + readonly IDataStreamFactory streamFactory; - public MainForm() + public MainForm(PrimitiveManager primitiveManager) + : this(primitiveManager, primitiveManager) { } + + public MainForm(PrimitiveManager primitiveManager, IDataStreamFactory streamFactory) { + this.primitiveManager = primitiveManager; + this.streamFactory = streamFactory; //Task.Run(async () => await _run()); InitializeComponent(); // myKQ = ConfigurationRequester.getApplicationQualifier(); @@ -109,11 +115,11 @@ namespace BCG private IDictionary myGetLibraryData() { SortedList retVal = new SortedList(); - string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames(); + string[] primitiveNames = primitiveManager.getPrimitiveNames(); foreach (string pName in primitiveNames) { - string aName = PrimitiveManager.instance.getAssemblyName(pName); - ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName); + string aName = primitiveManager.getAssemblyName(pName); + ICollection cNames = primitiveManager.getConstructorSigns(pName); if (!retVal.Contains(aName)) retVal.Add(aName, new SortedList()); (retVal[aName] as IDictionary)[pName] = new ArrayList(cNames); @@ -194,7 +200,7 @@ namespace BCG myExp = exp; myCrono.Restart(); - (myCurrentStream = myExp.getView(new Dictionary(0))).onChange += myCallback; + (myCurrentStream = myExp.getView(new Dictionary(0), streamFactory)).onChange += myCallback; myChangeStyle = true; //new SimpleBindableStream(myExp.getView(new Hashtable())).onChange += new DataStream.ChangeDelegate(myCallback); /* @@ -432,7 +438,7 @@ namespace BCG private void MainForm_Closed(object sender, System.EventArgs e) { if (myExp != null) - myExp.getView(new Dictionary(0)).onChange -= myCallback; + myExp.getView(new Dictionary(0), primitiveManager).onChange -= myCallback; } private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) @@ -667,7 +673,7 @@ namespace BCG DialogResult dr = ofd.ShowDialog(this); if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK) { - PrimitiveManager.instance.loadAssembly(ofd.FileName); + primitiveManager.loadAssembly(ofd.FileName); myLoadAssemblies(); } } diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs index 777f181f4..d7f102613 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs @@ -4,6 +4,7 @@ using bts; using bts.api.core; using bts.md; +using Bts.DataStreamCore; using Bts.DataStreamQueryAnalyzer; using Bts.PubSub; @@ -29,7 +30,14 @@ static class Program { await Configuration.InitializeAsync(cfg.Sparkd); PrimitiveManager.SetMarketDataClient(new MarketDataClient(new(TimeSpan.FromSeconds(0.5)))); Log.Gen.info("Starting DataStreamQueryAnalyzer"); - var mainForm = new MainForm(); + var primitiveManager = new PrimitiveManager( + new PrimitiveManagerConfig() { + Assemblies = new() { + "DataStream.dll", + "SharedStreams.dll" + }, + }); + var mainForm = new MainForm(primitiveManager); if (cfg.StreamFile is not null) { mainForm.loadFile(cfg.StreamFile); } diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesBookStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesBookStream.cs index 470e67c23..42cee34a5 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesBookStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesBookStream.cs @@ -1,13 +1,13 @@ namespace bts.BlueOrb { using BCG; - using System; + using bts.md; - [DataStreamUserRespresentation("LiveFuturesBooktream")] + [DataStreamUserRespresentation(false)] public class LiveFuturesBooktream : ViewStream { - public LiveFuturesBooktream(string future_target) { - ViewStream contract_str = new LiveFuturesContractStream(future_target); - ViewStream bbo_str = new BookStream(); + public LiveFuturesBooktream(string future_target, MarketDataClient mdClient) { + ViewStream contract_str = new LiveFuturesContractStream(future_target, mdClient); + ViewStream bbo_str = new BookStream(mdClient); ViewStream rs = new SimpleJoinStream(contract_str, "sec_key", bbo_str, "sec_key"); rs = new ProjectStream(rs, "left_sec:sec, left_sec_key:sec_key, bid_price, bid_sz:bid_size, ask_price, ask_sz:ask_size"); mySetRealSource(rs); diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesContractStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesContractStream.cs index 74b780cca..47119e999 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesContractStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesContractStream.cs @@ -4,13 +4,14 @@ using BCG; using bts.api; + using bts.md; [DataStreamUserRespresentation("LiveFuturesContractStream")] public class LiveFuturesContractStream : ViewStream { - public LiveFuturesContractStream(string target) { + public LiveFuturesContractStream(string target, MarketDataClient mdClient) { var fc = new ContractUtils.FutureContract(target); // temporary - ViewStream ins_str = new InstrumentsStream(); + ViewStream ins_str = new InstrumentsStream(mdClient); //new bts.BlueOrb.CSVReaderStream(@"C:\Temp\instruments.csv"); var rs = new LiveFuturesContractStream(ins_str, fc); mySetRealSource(rs); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbScoreStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbScoreStream.cs index 5d847ea59..b2217468b 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbScoreStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbScoreStream.cs @@ -1,15 +1,16 @@ namespace bts.BlueOrb { - using bts.SilverPine; using BCG; using System; + using bts.md; - [DataStreamUserRespresentation("SpatialArbScoreStream")] + [DataStreamUserRespresentation(false)] + /// This stream is now created by public class SpatialArbScoreStream : ViewStream { - public SpatialArbScoreStream(int window_sz_secs, int sample_period_sz) - : this(TimeSpan.FromSeconds(window_sz_secs), TimeSpan.FromSeconds(sample_period_sz), ViewStreamDatabase.instance) { } + public SpatialArbScoreStream(MarketDataClient mdClient, int window_sz_secs, int sample_period_sz) + : this(mdClient, TimeSpan.FromSeconds(window_sz_secs), TimeSpan.FromSeconds(sample_period_sz), ViewStreamDatabase.instance) { } - public SpatialArbScoreStream(TimeSpan window_sz, TimeSpan sample_period, ViewStreamDatabase vsd) { - ViewStream rs = new SpatialArbStatsStream(window_sz, sample_period, vsd); + public SpatialArbScoreStream(MarketDataClient mdClient, TimeSpan window_sz, TimeSpan sample_period, ViewStreamDatabase vsd) { + ViewStream rs = new SpatialArbStatsStream(mdClient, window_sz, sample_period, vsd); rs = new ThrottleStream(rs, TimeSpan.FromSeconds(1), vsd); rs = new SortStream(rs, "-n_trade_ticks"); mySetRealSource(rs); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs index 2985d8e81..c06b8ef25 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs @@ -1,17 +1,15 @@ namespace bts.BlueOrb { using BCG; - using SharpCompress.Common.Rar; + using bts.md; using System; using System.Collections.Generic; - using System.Linq; - using System.Text; - using System.Threading.Tasks; // For all the pairs generated by SpatialFuturePairsStream, this stream // generates the statistics, by appending together a SpreadStatsStream - [DataStreamUserRespresentation("SpatialArbStatsStream")] + /// This stream is now created by + [DataStreamUserRespresentation(false)] public class SpatialArbStatsStream : ViewStream{ private ViewStream _spatial_pairs_stream; private ViewStreamDatabase _vsd; @@ -20,14 +18,14 @@ private ViewStream _bbo_stream; private readonly TimeSpan _window_sz, _sampling_period_sz; - public SpatialArbStatsStream(int window_sz_seconds, int sampling_period_sz_seconds) - : this(TimeSpan.FromSeconds(window_sz_seconds), TimeSpan.FromSeconds(sampling_period_sz_seconds), ViewStreamDatabase.instance) { } + public SpatialArbStatsStream(MarketDataClient mdClient, int window_sz_seconds, int sampling_period_sz_seconds) + : this(mdClient, TimeSpan.FromSeconds(window_sz_seconds), TimeSpan.FromSeconds(sampling_period_sz_seconds), ViewStreamDatabase.instance) { } - public SpatialArbStatsStream(TimeSpan window_sz, TimeSpan sampling_period_sz, ViewStreamDatabase vsd) { + public SpatialArbStatsStream(MarketDataClient mdClient, TimeSpan window_sz, TimeSpan sampling_period_sz, ViewStreamDatabase vsd) { _vsd = vsd; _window_sz = window_sz; _sampling_period_sz = sampling_period_sz; - _spatial_pairs_stream = new SpatialFuturePairsStream(vsd); + _spatial_pairs_stream = new SpatialFuturePairsStream(mdClient, vsd); _bbo_stream = new BBOStream(vsd); } diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs index 4c59e7cea..084e92631 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs @@ -2,23 +2,22 @@ using BCG; using bts.api; + using bts.md; using System; using System.Collections.Generic; - using System.Linq; - using System.Text; - using System.Threading.Tasks; - [DataStreamUserRespresentation("SpatialFuturePairsStream")] + /// This stream is now created by + [DataStreamUserRespresentation(false)] public class SpatialFuturePairsStream : ViewStream { private ViewStream _main_stream; private IBlueTableSchema _schema; private HashSet _key_map = new HashSet(); - public SpatialFuturePairsStream() - : this(ViewStreamDatabase.instance) { } + public SpatialFuturePairsStream(MarketDataClient mdClient) + : this(mdClient, ViewStreamDatabase.instance) { } - public SpatialFuturePairsStream(ViewStreamDatabase vsd) { - _main_stream = new InstrumentsStream(vsd); + public SpatialFuturePairsStream(MarketDataClient mdClient, ViewStreamDatabase vsd) { + _main_stream = new InstrumentsStream(mdClient, vsd); _main_stream = new FilterStream(_main_stream, "sec_type='Future'"); _schema = new BlueTableSchema("SpatialFutures", new ColumnInfo("id", 0, TypeCode.Int32)); _schema = _schema.add_column(TypeCode.String, "s1"); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs index 24b3f24a6..1b25cd42a 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs @@ -18,7 +18,7 @@ namespace bts.BlueOrb{ using SecurityKey = System.UInt32; using System.Diagnostics; - [DataStreamUserRespresentation("books")] + [DataStreamUserRespresentation(false)] public class BookStream : ViewStream, IWorker { private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500); private DateTime _last_poll_time = DateTime.MinValue; @@ -39,14 +39,13 @@ namespace bts.BlueOrb{ IDisposable? subscription; // TODO: add cons for symbol, exchange etc - public BookStream() - : this(ViewStreamDatabase.instance) { - } + public BookStream(MarketDataClient mdClient) + : this(mdClient, ViewStreamDatabase.instance) { } - public BookStream(ViewStreamDatabase vsd) { + public BookStream(MarketDataClient mdClient, ViewStreamDatabase vsd) { bts.Log.Gen.info("BookStream.cons"); _vsd = vsd; - _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5))); + _md_client = mdClient; } protected override void myOnFirstSubscriber() { diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs index 4a83b24cb..6fb5c4a4c 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs @@ -43,14 +43,13 @@ namespace bts.BlueOrb { IDisposable? subscription; // TODO: add cons for symbol, exchange etc - public InstrumentsStream() - : this(ViewStreamDatabase.instance) { - } + public InstrumentsStream(MarketDataClient mdClient) + : this(mdClient, ViewStreamDatabase.instance) { } - public InstrumentsStream(ViewStreamDatabase vsd) { + public InstrumentsStream(MarketDataClient mdClient, ViewStreamDatabase vsd) { bts.Log.Gen.info("InstrumentsStream.cons"); _vsd = vsd; - _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5))); + _md_client = mdClient; } protected override void myOnFirstSubscriber() { diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs index ae5523fb3..4713fe802 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs @@ -213,7 +213,14 @@ namespace bts.BlueOrb { } - - + internal OHLCVStream(ViewStream source, OHLCVBarSize bar_size) { + ViewStream rs = source; + rs = new SchemaGuesserStream(rs); + if (bar_size == OHLCVBarSize.ONE_DAY) { + rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime); + } + rs = new NewProjectStream(rs, "symbol:contract"); + mySetRealSource(rs); + } } } diff --git a/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLStream.cs b/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLStream.cs index 330dfc309..cf40a0cd4 100644 --- a/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLStream.cs +++ b/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLStream.cs @@ -1,19 +1,23 @@ namespace bts.BlueOrb { using BCG; + using Bts.DataStreamCore; + using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; - [DataStreamUserRespresentation("nl")] + [DataStreamUserRespresentation(false)] public class BlueOrbNLStream : ViewStream { private readonly string _nl_prompt; + readonly IDataStreamFactory streamFactory; private ViewStream _nl_to_dsl_stream, _dsl_stream; private IDictionary _dp_metadata = new Dictionary(); - public BlueOrbNLStream(string nl_prompt) { + public BlueOrbNLStream(string nl_prompt, IDataStreamFactory streamFactory) { _nl_prompt = nl_prompt; + this.streamFactory = streamFactory; } protected override void myOnFirstSubscriber() { @@ -27,7 +31,7 @@ string dsl = r.get("response"); _dp_metadata["dsl"] = dsl; Log.Gen.info($"NLStream._handle_dsl_callback dsl: {dsl}"); - _dsl_stream = new DSLStream(dsl); + _dsl_stream = new DSLStream(dsl, streamFactory); _dsl_stream.onChange += _handle_data_cb; } diff --git a/src/DataStreamSuite/SharedStreams/NL/NLPromptConfig.cs b/src/DataStreamSuite/SharedStreams/NL/NLPromptConfig.cs new file mode 100644 index 000000000..27faaa1bb --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/NL/NLPromptConfig.cs @@ -0,0 +1,77 @@ +namespace bts.SilverPine; + +using Spark.Core.Json; + +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +public class NLPromptConfig { + static NLPromptConfig? instance; + + public static NLPromptConfig Instance => + instance ?? throw new InvalidOperationException( + "SetGlobalConfig must be called to initialize config static singleton"); + + public List prompts { get; } = new(0); + + public static void SetGlobalConfig(NLPromptConfig cfg) { + // This should only be called once + Log.isnull(instance); + instance = Log.notnull(cfg); + } + + public static NLPromptConfig Default() => new(); + + // Represents a pair of a natural language query and its dsl + public class FewShotExample { + public string? natural_language { get; set; } + public string? dsl { get; set; } + } + + public class FewShotPrompt { + public string? category { get; set; } + + public string? description { get; set; } + + public bool IsEnabled { get; set; } = true; + + // MultiLineDescription is simply here for convenience. It is functionally redundant, as it + // serves the same purpose as 'description'. If 'description' is too long, you can break it + // up into an array of strings and assign it to MultiLineDescription instead. + public List? MultiLineDescription { get; set; } + + // the actual prompt given by the user/CME + public string? prompt { get; set; } + + public List examples { get; } = new(0); + + public override string ToString() { + var desc = description ?? string.Concat(MultiLineDescription.empty_if_null()); + string rv = $"System: {desc}\n"; + int ex_no = 0; + foreach (FewShotExample ex in examples ?? new(0)) { + rv += $"{++ex_no}. Natural Language: \"{ex.natural_language}\"" + "\n"; + rv += $" DSL: {ex.dsl}\n"; + + } + return rv; + } + + // Reads normal strings and arrays of lines + public class FewShotPromptLinesConverter : JsonConverter { + public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (reader.ReadString(out var str)) { + return str; + } + if (reader.ReadArray(out var array)) { + return string.Join('\n', array.Select(v => v?.AsString())); + } + throw new JsonException("Failed to parse FewShotPrompt lines"); + } + + public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) => + writer.WriteStringValue(value); + } + } +} diff --git a/src/DataStreamSuite/SharedStreams/NL/NLStreamFactory.cs b/src/DataStreamSuite/SharedStreams/NL/NLStreamFactory.cs new file mode 100644 index 000000000..5e21ada4a --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/NL/NLStreamFactory.cs @@ -0,0 +1,44 @@ +namespace bts.SilverPine; + +using BCG; + +using bts.BlueOrb; + +using Bts.DataStreamCore; + +using mbts.utils; + +using System.Collections.Generic; + +public class NLStreamFactory : IRecursiveDataStreamFactory { + IDataStreamFactory? compositeFactory; + + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (SupportedStreams.NaturalLanguage.IsMatch(streamId)) { + if (compositeFactory is null) { + Log.DataFlow.warn("Failed to create NLStream because 'compositeFactory' is null."); + return null; + } + if (args.SingleOrDefault().Value is string prompt) { + return new BlueOrbNLStream(prompt, compositeFactory); + } + } + return null; + } + + public IEnumerable GetSupportedStreams() { + yield return SupportedStreams.NaturalLanguage; + } + + public void OnCompositeFactory(IDataStreamFactory compositeFactory) => + this.compositeFactory = compositeFactory; + + static class SupportedStreams { + public static readonly DataStreamMetadata NaturalLanguage = new() { + Name = "nl", + AlternateNames = new() { "naturalLanguage" } + }; + + public static string NlName => ExceptionAssert.not_null(NaturalLanguage.Name); + } +} diff --git a/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs b/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs new file mode 100644 index 000000000..340b7a9bd --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs @@ -0,0 +1,28 @@ +namespace Bts.SharedStreams.Services; + +using bts.SilverPine; + +using Bts.DataStreamCore; +using Bts.SharedStreams.StreamFactories; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +public static class SharedStreamsServices { + public static IServiceCollection AddMarketDataStreams(this IServiceCollection services) => + services.addStreamFactory() + .addStreamFactory(); + + public static IServiceCollection AddNaturalLanguageStreams(this IServiceCollection services, IConfigurationSection configurationSection) { + services.Configure(configurationSection); + NLPromptConfig.SetGlobalConfig(configurationSection.Get() ?? NLPromptConfig.Default()); + // TODO: integrate config into a NLStream provider (so it can provide config values + // instead of the ViewStream's getting them from a static singleton) + return services.AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()); + } + + static IServiceCollection addStreamFactory(this IServiceCollection services) where T : class, IDataStreamFactory => + services.AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()); +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/BookStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/BookStreamFactory.cs new file mode 100644 index 000000000..36b68126a --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/BookStreamFactory.cs @@ -0,0 +1,32 @@ +namespace Bts.SharedStreams.StreamFactories; + +using BCG; + +using bts.md; + +using Bts.DataStream; +using Bts.DataStreamCore; +using Bts.SharedStreams.MarketData; + +using Microsoft.Extensions.Logging; + +using System; + +public class BookStreamFactory : DataStreamFactory { + public BookStreamFactory(MarketDataClient? mdClient, ILogger logger) { + this.Add("book", CreateBook, logger); + this.Add(new[] { "bbo", "LiveBBOStream" }, CreateBbo, logger); + + ViewStream? CreateBook(BookStreamParams streamParams) => + mdClient?.books(streamParams.GetSecurityFilter()) + ?.AsDataPackets(streamParams.GetSchemaBuilder().BookTupleAdapter()) + ?.AsViewStream() + ?? throw new InvalidOperationException("Live md not available"); + + ViewStream? CreateBbo(BookStreamParams streamParams) => + mdClient?.books(streamParams.GetSecurityFilter()) + ?.AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter()) + ?.AsViewStream() + ?? throw new InvalidOperationException("Live md not available"); + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/InstrumentSchemaBuilder.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/InstrumentSchemaBuilder.cs new file mode 100644 index 000000000..25d95f120 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/InstrumentSchemaBuilder.cs @@ -0,0 +1,78 @@ +namespace Bts.SharedStreams.StreamFactories; + +using BCG; + +using bts.api; + +using Bts.DataStreamCore; + +public class InstrumentSchemaBuilder { + public bool ShowVenue { get; set; } = true; + public bool ShowAsset { get; set; } = true; + public bool ShowType { get; set; } = true; + public bool ShowName { get; set; } = true; + public bool ShowId { get; set; } = true; + public bool ShowCashPerPoint { get; set; } = true; + public bool ShowLegId { get; set; } = true; + public bool ShowLegVenue { get; set; } + public bool ShowLegAsset { get; set; } + public bool ShowLegType { get; set; } + public bool ShowLegName { get; set; } = true; + public bool ShowLegCashPerPoint { get; set; } + public int NumLegs { get; set; } = 2; + public string TableName { get; set; } = "Instrument"; + + public BlueTableSchemaAdapter SecurityAdapter() { + var adapter = new BlueTableSchemaAdapter(TableName); + adapter.AddColumn("id", r => (int)r.Id); + if (ShowVenue) { + adapter.AddColumn(WellKnownColumns.Venue.Name, b => b.Exchange.ToString()); + } + if (ShowType) { + adapter.AddColumn(WellKnownColumns.SecurityType.Name, b => b.security_type().ToString()); + } + if (ShowAsset) { + adapter.AddColumn(WellKnownColumns.Asset.Name, b => b.Symbol); + } + if (ShowName) { + adapter.AddColumn(WellKnownColumns.SecurityName.Name, b => b.ToString()); + } + if (ShowId) { + adapter.AddColumn(WellKnownColumns.SecurityId.Name, b => b.Id); + } + if (ShowCashPerPoint) { + adapter.AddColumn(WellKnownColumns.CashPerPoint.Name, b => b.ContractSpec.CashPerPoint); + } + for (int i = 0; i < NumLegs; i++) { + if (ShowLegVenue) { + adapter.AddColumn(WellKnownColumns.LegVenue(i).Name, Get(l => l.Security.Exchange.ToString(), null)); + } + if (ShowLegType) { + adapter.AddColumn(WellKnownColumns.LegSecurityType(i).Name, Get(l => l.Security.security_type().ToString(), null)); + } + if (ShowLegAsset) { + adapter.AddColumn(WellKnownColumns.LegAsset(i).Name, Get(l => l.Security.Symbol, null)); + } + if (ShowLegName) { + adapter.AddColumn(WellKnownColumns.LegSecurityName(i).Name, Get(l => l.Security.ToString(), null)); + } + if (ShowLegId) { + adapter.AddColumn(WellKnownColumns.LegSecurityId(i).Name, Get(l => (int)l.Security.Id, 0)); + } + if (ShowLegCashPerPoint) { + adapter.AddColumn(WellKnownColumns.LegCashPerPoint(i).Name, Get(l => l.Security.ContractSpec.CashPerPoint, double.NaN)); + } + + Func Get(Func getValue, T defaultValue) => + s => { + ContractLegImmutable? leg = s.ContractCase switch { + Security.ContractOneofCase.Spread => s.Spread.Legs.Skip(i).TryGetFirst(out var l) ? l : null, + Security.ContractOneofCase.Synthetic => s.Synthetic.Legs.Skip(i).TryGetFirst(out var l) ? l : null, + _ => null, + }; + return leg.HasValue ? getValue(leg.Value) : defaultValue; + }; + } + return adapter; + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/InstrumentStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/InstrumentStreamFactory.cs new file mode 100644 index 000000000..524cb62f4 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/InstrumentStreamFactory.cs @@ -0,0 +1,34 @@ +namespace Bts.SharedStreams.StreamFactories; + +using BCG; + +using bts.api.core; +using bts.BlueOrb; +using bts.md; + +using Bts.DataStream; +using Bts.DataStreamCore; +using Bts.SharedStreams.MarketData; + +using Microsoft.Extensions.Logging; + +class InstrumentStreamFactory : DataStreamFactory { + public InstrumentStreamFactory(MarketDataClient mdClient, ILogger logger) { + this.Add("instrument", CreateInstrument, logger); + this.Add(new[] { "liveFutures", "liveFuturesContract", "LiveFuturesContractStream" }, CreateLiveFutureContract, logger); + + ViewStream? CreateInstrument(InstrumentStreamParams streamParams) => + mdClient.securities(streamParams.GetSecurityCriteria()) + .Table(s => s.Id) + .AsDataPackets(streamParams.GetSchemaBuilder().SecurityAdapter()) + .AsViewStream(); + + ViewStream? CreateLiveFutureContract(LiveFutureContractStreamParams streamParams) => + new LiveFuturesContractStream(streamParams.Target ?? string.Empty, mdClient); + } + + public class LiveFutureContractStreamParams { + [StreamArgument(RegexNames = new[] { "^(symbol|sym|product|asset|target|tgt)$" })] + public string? Target { get; set; } + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/InstrumentStreamParams.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/InstrumentStreamParams.cs new file mode 100644 index 000000000..f9010b282 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/InstrumentStreamParams.cs @@ -0,0 +1,82 @@ +namespace Bts.SharedStreams.StreamFactories; + +using bts.api; + +using Bts.DataStreamCore; + +public class InstrumentStreamParams { + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.VenueRegex })] + public Venue? Venue { get; set; } + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.AssetRegex })] + public string? Asset { get; set; } + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityType })] + public SecurityType? Type { get; set; } + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityName })] + public string? Name { get; set; } + + [StreamArgument("legs", "nLegs", "numLegs")] + public int NumLegs { get; set; } = 5; + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowVenueRegex })] + public bool ShowVenue { get; set; } = true; + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowAssetRegex })] + public bool ShowAsset { get; set; } = true; + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowSecurityTypeRegex })] + public bool ShowType { get; set; } = true; + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowSecurityNameRegex })] + public bool ShowName { get; set; } = true; + + public bool ShowId { get; set; } = false; + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowCashPerPointRegex })] + public bool ShowCashPerPoint { get; set; } = true; + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegVenueRegex })] + public bool ShowLegVenue { get; set; } = true; + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegAssetRegex })] + public bool ShowLegAsset { get; set; } = true; + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegSecurityTypeRegex })] + public bool ShowLegType { get; set; } = true; + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegNameRegex })] + public bool ShowLegName { get; set; } = true; + + public bool ShowLegId { get; set; } = false; + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegCashPerPointRegex })] + public bool ShowLegCashPerPoint { get; set; } = true; + + public mbts.spark.SecurityCriteria GetSecurityCriteria() { + var clause = new mbts.spark.SecurityCriteriaClause( + exchanges: Venue.HasValue ? new[] { Venue.Value } : null, + security_types: Type.HasValue ? new[] { Type.Value } : null, + symbols: !string.IsNullOrEmpty(Asset) ? new[] { Asset } : null, + names: !string.IsNullOrEmpty(Name) ? new[] { Name } : null); + return clause.is_defined ? new(new[] { clause }) : new(); + } + + public InstrumentSchemaBuilder GetSchemaBuilder() => + new() { + NumLegs = NumLegs, + ShowVenue = ShowVenue, + ShowType = ShowType, + ShowAsset = ShowAsset, + ShowName = ShowName, + ShowId = ShowId, + ShowCashPerPoint = ShowCashPerPoint, + ShowLegVenue = ShowLegVenue, + ShowLegType = ShowLegType, + ShowLegAsset = ShowLegAsset, + ShowLegName = ShowLegName, + ShowLegId = ShowLegId, + ShowLegCashPerPoint = ShowLegCashPerPoint, + }; +} diff --git a/src/client/spark_api/MarketData/BookTuple.cs b/src/client/spark_api/MarketData/BookTuple.cs index db7074cae..11c035931 100644 --- a/src/client/spark_api/MarketData/BookTuple.cs +++ b/src/client/spark_api/MarketData/BookTuple.cs @@ -3,8 +3,10 @@ namespace Spark.Api.MarketData { using bts.api; using bts.api.domain; using bts.md; + using bts.utils; using System; + using System.Linq; public readonly struct BookTuple : IEquatable, IJsonFormatter { readonly TickDisplaySettingsImmutable? _quantity_display; @@ -19,9 +21,13 @@ namespace Spark.Api.MarketData { _quantity_display = quantity_display; } + public static BookTuple Empty(SecurityImmutable security) => + new(security, Book.empty(security.Id)); + public SecurityImmutable security { get; } public uint security_id => security.Id; public Book book { get; } + public Bbo bbo { get { (double bidp, double bidq) = book.bids.Count > 0 ? (book.bids[0].price, book.bids[0].size) : (double.NaN, double.NaN); @@ -33,6 +39,8 @@ namespace Spark.Api.MarketData { public TickDisplaySettingsImmutable quantity_display => _quantity_display ?? Security.Defaults.quantity_display_settingsi; + public DateTime? Timestamp { get; init; } + public bool Equals(BookTuple other) => security.Equals(other.security) && book.Equals(other.book); @@ -52,4 +60,15 @@ namespace Spark.Api.MarketData { public static bool operator !=(BookTuple left, BookTuple right) => !(left == right); } + + public static class BookTupleExtensions { + public static Book NotionalBook(this BookTuple bookTuple) => + bookTuple.book.ToNotional(bookTuple.security.ContractSpec.CashPerPoint); + + public static Book ToNotional(this Book book, double cpp) => + new( + book.security_id, + book.bids.Select(b => new PriceSize((float)(b.size * cpp * b.price), b.price)).ToList(), + book.asks.Select(a => new PriceSize((float)(a.size * cpp * a.price), a.price)).ToList()); + } } diff --git a/src/client/spark_api/MarketData/MarketDataClient.cs b/src/client/spark_api/MarketData/MarketDataClient.cs index ccc5120ae..780813659 100644 --- a/src/client/spark_api/MarketData/MarketDataClient.cs +++ b/src/client/spark_api/MarketData/MarketDataClient.cs @@ -60,7 +60,7 @@ namespace bts.md { } public IObservable> books(SecurityCriteria? criteria = null) => - books(criteria is null ? null : + books(criteria is null || !criteria.is_defined() ? null : new Func(s => criteria.includes(s))); public IObservable> books(Func? filter) { diff --git a/src/client/spark_api/MarketData/PriceSize.cs b/src/client/spark_api/MarketData/PriceSize.cs index 42d04c3af..5a59d6acd 100644 --- a/src/client/spark_api/MarketData/PriceSize.cs +++ b/src/client/spark_api/MarketData/PriceSize.cs @@ -2,14 +2,12 @@ namespace bts.md { using System; using Size = System.Single; - public class PriceSize - { + public class PriceSize { public Size size { get; } public Size direct_size { get; } public Size implied_size { get; } public double price { get; } - public PriceSize(Size sz, Size direct_sz, Size implied_sz, double prx) - { + public PriceSize(Size sz, Size direct_sz, Size implied_sz, double prx) { size = sz; direct_size = direct_sz; implied_size = implied_sz; diff --git a/src/client/spark_api/MarketData/SyntheticBook.cs b/src/client/spark_api/MarketData/SyntheticBook.cs new file mode 100644 index 000000000..185d4783b --- /dev/null +++ b/src/client/spark_api/MarketData/SyntheticBook.cs @@ -0,0 +1,26 @@ +#nullable enable +using bts.api; + +using Spark.Api.MarketData; + +using System.Collections.Generic; + +public readonly struct SyntheticBook { + public SyntheticBook( + uint securityId, IReadOnlyList bids, IReadOnlyList asks) { + SecurityId = securityId; + Bids = bids; + Asks = asks; + } + + public uint SecurityId { get; } + public IReadOnlyList Bids { get; } + public IReadOnlyList Asks { get; } + + public SyntheticBookLevel this[Side side, int level] => + side switch { + Side.Buy => level < Bids.Count ? Bids[level] : new(double.NaN, double.NaN, double.NaN), + Side.Sell => level < Asks.Count ? Asks[level] : new(double.NaN, double.NaN, double.NaN), + _ => throw new KeyNotFoundException("Invalid side"), + }; +} diff --git a/src/client/spark_api/MarketData/SyntheticBookCalculator.cs b/src/client/spark_api/MarketData/SyntheticBookCalculator.cs index fd70cc6fd..da87f931e 100644 --- a/src/client/spark_api/MarketData/SyntheticBookCalculator.cs +++ b/src/client/spark_api/MarketData/SyntheticBookCalculator.cs @@ -3,6 +3,7 @@ namespace Spark.Api.MarketData; using bts; using bts.api; +using bts.collections; using bts.md; using bts.utils; @@ -18,7 +19,7 @@ public static class SyntheticBookCalculator { GetSyntheticDepth(legDepths.Select(GetBidLegParams).ToList()).Select(ToPriceSize).ToList(), GetSyntheticDepth(legDepths.Select(GetAskLegParams).ToList()).Select(ToPriceSize).ToList()); - PriceSize ToPriceSize(DepthLevel level) => + static PriceSize ToPriceSize(DepthLevel level) => new((float)level.Quantity, level.Price); (double Quantity, double TickSize, Book Book) GetBidLegParams((ContractLegImmutable, Book) ld) => @@ -28,6 +29,22 @@ public static class SyntheticBookCalculator { (-ld.Item1.signed_quantity(), ld.Item1.Security.ContractSpec.LowTick, ld.Item2); } + public static Book GetSyntheticBook(uint securityId, IEnumerable<(double Quantity, double TickSize, Book Book)> legDepths) { + return new( + securityId, + GetSyntheticDepth(legDepths.ToList()).Select(ToPriceSize).ToList(), + GetSyntheticDepth(legDepths.Select(l => (-l.Quantity, l.TickSize, l.Book)).ToList()).Select(ToPriceSize).ToList()); + + static PriceSize ToPriceSize(DepthLevel level) => + new((float)level.Quantity, level.Price); + } + + public static SyntheticBook GetSyntheticNotionalBook(uint securityId, IEnumerable<(double Quantity, double TickSize, double CashPerPoint, Book Book)> legDepths) => + new( + securityId, + GetSyntheticNotionalDepth(legDepths.ToList()).ToList(), + GetSyntheticNotionalDepth(legDepths.Select(l => (-l.Quantity, l.TickSize, l.CashPerPoint, l.Book)).ToList()).ToList()); + public static List GetSyntheticDepth(IEnumerable<(ContractLegImmutable, Book)> legDepths) { return GetSyntheticDepth( legDepths.Select(getLegDepthConsumer).ToList()); @@ -37,15 +54,18 @@ public static class SyntheticBookCalculator { Log.assert(leg.Quantity.is_real(), "Leg quantity must be real"); Log.assert(leg.Security.HasContractSpec && leg.Security.ContractSpec.LowTick.is_real_positive(), "Leg security LowTick must be real and positive"); - return LegDepthConsumer.Create(leg.Quantity, leg.Security.ContractSpec.LowTick); + var depthLevels = (leg.Side == Side.Buy ? book.bids : book.asks) + .Select(lvl => new DepthLevel(lvl.price, lvl.size)) + .ToArray(); + return LegDepthConsumer.Create(leg.Quantity, leg.Security.ContractSpec.LowTick, depthLevels); } } public static List GetSyntheticDepth(IEnumerable<(double Quantity, double TickSize, Book Book)> legDepths) { return GetSyntheticDepth( - legDepths.Select(getLegDepthConsumer).ToList()); + legDepths.Select(GetLegDepthConsumer).ToList()); - LegDepthConsumer getLegDepthConsumer((double Quantity, double TickSize, Book Book) legDepth) { + LegDepthConsumer GetLegDepthConsumer((double Quantity, double TickSize, Book Book) legDepth) { var (qty, tickSz, bk) = legDepth; Log.assert(qty.is_real(), "Leg quantity must be real"); Log.assert(tickSz.is_real_positive(), @@ -54,6 +74,29 @@ public static class SyntheticBookCalculator { } } + public static List GetSyntheticNotionalDepth(IEnumerable<(double Quantity, double TickSize, double CashPerPoint, Book Book)> legDepths) { + var legs = legDepths.Select(InitializeLegDepth).ToList(); + if (legs.Count == 0) { + return new(0); + } + return legs.Skip(1).Aggregate(legs[0], AggregateLegDepth).Depth.Levels; + + SyntheticLegDepthConsumer InitializeLegDepth((double Quantity, double TickSize, double CashPerPoint, Book Book) legDepth) { + var (qty, tickSz, cashPerPoint, bk) = legDepth; + Log.assert(qty.is_real(), "Leg quantity must be real"); + Log.assert(tickSz.is_real_positive(), + "Leg tickSz must be real and positive"); + var levels = (qty > 0 ? bk.bids : bk.asks).ToArray(l => new SyntheticBookLevel(l.price, l.size, l.NotionalValue(cashPerPoint))); + return SyntheticLegDepthConsumer.Create(qty, tickSz, levels); + } + + SyntheticLegDepthConsumer AggregateLegDepth(SyntheticLegDepthConsumer l1, SyntheticLegDepthConsumer l2) => + new(new(getSyntheticNotionalDepth(l1, l2).ToList())) { + LegQuantity = 1, + QuantityTick = MathUtils.lcm(l1.QuantityTick, l2.QuantityTick), + }; + } + public static List GetSyntheticDepth(List legs) { if (legs.Count == 0) { return new(0); @@ -90,6 +133,39 @@ public static class SyntheticBookCalculator { } } + static IEnumerable getSyntheticNotionalDepth( + SyntheticLegDepthConsumer leg1, SyntheticLegDepthConsumer leg2) { + while (!leg1.Depth.IsDone && !leg2.Depth.IsDone) { + if (!leg1.LegQuantity.is_real() || !leg2.LegQuantity.is_real()) { + throw new ArgumentException("Invalid leg quantities"); + } + double leg2PerLeg1 = (leg2.LegQuantity / leg1.LegQuantity).abs(); + double leg1PerLeg2 = (leg1.LegQuantity / leg2.LegQuantity).abs(); + double avail1 = leg1.Depth.AvailableQuantity(1); + double avail2 = leg2.Depth.AvailableQuantity(1); + (double take1, double take2) = (avail1 / leg1PerLeg2) < (avail2 / leg2PerLeg1) + ? (avail1, avail1 * leg2PerLeg1) + : (avail2 * leg1PerLeg2, avail2); + var (qty1, cost1, notional1) = leg1.Depth.ConsumeQuantity(take1); + var (qty2, cost2, notional2) = leg2.Depth.ConsumeQuantity(take2); + Log.assert(qty2 / qty1 == leg2PerLeg1); + var price1 = cost1 / qty1; + var price2 = cost2 / qty2; + price1 *= leg1.LegQuantity; + price2 *= leg2.LegQuantity; + if (leg1.LegQuantity < 0) { + notional1 = -notional1; + } + if (leg2.LegQuantity < 0) { + notional2 = -notional2; + } + yield return new( + Price: price1 + price2, + Quantity: qty1, + NotionalValue: notional1 + notional2); + } + } + public record DepthLevel(double Price, double Quantity) { public override string ToString() { return $"{Price} ({Quantity})"; @@ -173,6 +249,7 @@ public static class SyntheticBookCalculator { public double LegQuantity { get; set; } = 1; public double QuantityTick { get; set; } = 1; + public double CashPerPoint { get; set; } = 1; public DepthConsumer Depth { get; } = new(); public static LegDepthConsumer Create(double legQuantity, double quantityTick, params DepthLevel[] levels) => diff --git a/src/client/spark_api/MarketData/SyntheticBookLevel.cs b/src/client/spark_api/MarketData/SyntheticBookLevel.cs new file mode 100644 index 000000000..6eaf2da58 --- /dev/null +++ b/src/client/spark_api/MarketData/SyntheticBookLevel.cs @@ -0,0 +1,12 @@ +#nullable enable + +namespace Spark.Api.MarketData; + +/// +/// Like a normal BookLevel, but includes the NotionalValue, as that could only +/// otherwise be computed with the leg quantities and cash per point. +/// +/// +/// +/// +public readonly record struct SyntheticBookLevel(double Price, double Quantity, double NotionalValue); diff --git a/src/client/spark_api/MarketData/SyntheticDepthConsumer.cs b/src/client/spark_api/MarketData/SyntheticDepthConsumer.cs new file mode 100644 index 000000000..2d2c973ab --- /dev/null +++ b/src/client/spark_api/MarketData/SyntheticDepthConsumer.cs @@ -0,0 +1,73 @@ +#nullable enable +namespace Spark.Api.MarketData; + +using System; +using System.Collections.Generic; +using System.Linq; + +public class SyntheticDepthConsumer { + public SyntheticDepthConsumer() { } + public SyntheticDepthConsumer(params SyntheticBookLevel[] levels) : this(levels.AsEnumerable()) { } + public SyntheticDepthConsumer(IEnumerable levels) => + Levels.AddRange(levels); + + public int Pos { get; private set; } = 0; + public double CurrLevelConsumed { get; private set; } = 0; + public List Levels { get; } = new(); + + public bool IsDone => + Pos == Levels.Count || (Pos == Levels.Count - 1 && Levels.Last().Quantity == CurrLevelConsumed); + + public double AvailableQuantity(int levelLimit) { + int end = Math.Min(Pos + levelLimit, Levels.Count); + double rv = 0; + for (int i = Pos; i < end; i++) { + double levelQty = Levels[i].Quantity; + if (i == Pos) { + levelQty -= CurrLevelConsumed; + } + rv += levelQty; + } + return rv; + } + + public (double Qty, double Price, double Notional) ConsumeQuantity(double quantity) { + double consumed = 0; + double consumedNotional = 0; + double price = 0; + for (; Pos < Levels.Count; Pos++) { + var level = Levels[Pos]; + double levelQty = level.Quantity - CurrLevelConsumed; + var rem = quantity - consumed; + if (levelQty > rem) { + CurrLevelConsumed += rem; + consumed = quantity; + var remCost = rem * Levels[Pos].Price; + price += remCost; + consumedNotional += Levels[Pos].NotionalValue * rem / Levels[Pos].Quantity; + break; + } + consumed += levelQty; + price += levelQty * Levels[Pos].Price; + consumedNotional += Levels[Pos].NotionalValue * levelQty / Levels[Pos].Quantity; + CurrLevelConsumed = 0; + } + return (consumed, price, consumedNotional); + } +} + +public class SyntheticLegDepthConsumer { + public SyntheticLegDepthConsumer() : this(new()) { } + public SyntheticLegDepthConsumer(SyntheticDepthConsumer depth) => + Depth = depth; + + public double LegQuantity { get; set; } = 1; + public double QuantityTick { get; set; } = 1; + public SyntheticDepthConsumer Depth { get; } = new(); + + public static SyntheticLegDepthConsumer Create(double legQuantity, double quantityTick, params SyntheticBookLevel[] levels) => + new(new(levels)) { + LegQuantity = legQuantity, + QuantityTick = quantityTick, + }; +} diff --git a/src/client/spark_api/data/SecurityCriteriaExtensions.cs b/src/client/spark_api/data/SecurityCriteriaExtensions.cs index bf1a11ba8..ee3e6ac94 100644 --- a/src/client/spark_api/data/SecurityCriteriaExtensions.cs +++ b/src/client/spark_api/data/SecurityCriteriaExtensions.cs @@ -108,10 +108,19 @@ namespace mbts.spark { security_criteria.is_defined() && security_criteria.clauses.empty_if_null().Any(c => c.includes(security)); + public static bool includes( + this SecurityCriteria security_criteria, (Venue? Venue, SecurityType? Type, string? Symbol) tup) => + security_criteria.is_defined() && + security_criteria.clauses.empty_if_null().Any(c => c.includes(tup)); + public static bool includes( this SecurityCriteriaClause clause, SecurityImmutable security) => _to_predicates(clause).AnyAndAll(p => p(security)); + public static bool includes( + this SecurityCriteriaClause clause, (Venue? Venue, SecurityType? Type, string? Symbol) tup) => + _to_symbol_predicates(clause).AnyAndAll(p => p(tup)); + public static bool is_subset_of( this SecurityCriteriaClause clause, SecurityCriteriaClause other) => // exchanges @@ -178,6 +187,10 @@ namespace mbts.spark { ) ); + public static bool includes(this SecurityCriteria security_criteria, (Venue Venue, SecurityType Type, string? symbol) sec_tup) => + security_criteria.is_defined() && + security_criteria.clauses.empty_if_null().Any(c => c.includes(sec_tup)); + public static SecurityCriteriaClause with(this SecurityCriteriaClause clause, Venue[]? exchanges = null, SecurityType[]? security_types = null, @@ -472,5 +485,17 @@ namespace mbts.spark { yield return sec => clause.names.Contains(sec.ToString()); } } + + static IEnumerable> _to_symbol_predicates(SecurityCriteriaClause clause) { + if (!clause.exchanges.is_null_or_empty()) { + yield return tup => !tup.Venue.HasValue || clause.exchanges.Contains(tup.Venue.Value); + } + if (!clause.symbols.is_null_or_empty()) { + yield return tup => tup.Symbol is null || clause.symbols.Contains(tup.Symbol); + } + if (!clause.security_types.is_null_or_empty()) { + yield return tup => !tup.Type.HasValue || clause.security_types.Contains(tup.Type.Value); + } + } } } diff --git a/src/client/spark_core/Json/JsonMultiLineString.cs b/src/client/spark_core/Json/JsonMultiLineString.cs new file mode 100644 index 000000000..9c0321018 --- /dev/null +++ b/src/client/spark_core/Json/JsonMultiLineString.cs @@ -0,0 +1,9 @@ +#nullable enable +namespace Spark.Core.Json; + +using System.Text.Json.Serialization; + +[JsonConverter(typeof(JsonMultiLineStringConverter))] +public class JsonMultiLineString { + public string? Value { get; set; } +} diff --git a/src/client/spark_core/Json/JsonMultiLineStringConverter.cs b/src/client/spark_core/Json/JsonMultiLineStringConverter.cs new file mode 100644 index 000000000..fc786364f --- /dev/null +++ b/src/client/spark_core/Json/JsonMultiLineStringConverter.cs @@ -0,0 +1,39 @@ +#nullable enable +namespace Spark.Core.Json; + +using System; + +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +// Reads normal strings and arrays of lines +public class JsonMultiLineStringConverter : JsonConverter { + public override JsonMultiLineString? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (reader.ReadString(out var str)) { + return new() { Value = str }; + } + if (reader.ReadArray(out var array)) { + return new() { Value = string.Join('\n', array.Select(v => v?.AsString())) }; + } + throw new JsonException("Failed to parse multi-line string"); + } + + public override void Write(Utf8JsonWriter writer, JsonMultiLineString value, JsonSerializerOptions options) => + writer.WriteStringValue(value.Value); +} + +public class JsonMultiLineStringValueConverter : JsonConverter { + public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (reader.ReadString(out var str)) { + return str; + } + if (reader.ReadArray(out var array)) { + return str; + } + throw new JsonException("Failed to parse multi-line string"); + } + + public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) => + writer.WriteStringValue(value); +} diff --git a/src/client/spark_core/Update/TableUpdate.cs b/src/client/spark_core/Update/TableUpdate.cs index 888ddcfda..ff6b971e1 100644 --- a/src/client/spark_core/Update/TableUpdate.cs +++ b/src/client/spark_core/Update/TableUpdate.cs @@ -3,6 +3,7 @@ namespace bts.api.core { using System; using System.Collections.Generic; using System.Collections.Immutable; + using System.Diagnostics.CodeAnalysis; using System.Linq; public class TableUpdate where TKey : IEquatable { @@ -277,6 +278,17 @@ namespace bts.api.core { _removed[kv.Key] = kv.Value; } + public bool TryGetValue(TKey key, [MaybeNullWhen(false)]out T result) { + if(_added.TryGetValue(key, out result)) { + return true; + } + if (_changes.TryGetValue(key, out var ch)) { + result = ch.New; + return true; + } + return !_removed.ContainsKey(key) && _state.state.TryGetValue(key, out result); + } + /// /// Creates a TableUpdate instance from the current state /// diff --git a/src/client/spark_core/utils/EnumHelper.cs b/src/client/spark_core/utils/EnumHelper.cs index b1ca947ce..d570e3cf2 100644 --- a/src/client/spark_core/utils/EnumHelper.cs +++ b/src/client/spark_core/utils/EnumHelper.cs @@ -68,6 +68,9 @@ namespace mbts.ui.utility { return Activator.CreateInstance(enum_type); } + public static T? TryParse(string value, bool ignoreCase = true) where T : struct, Enum => + Enum.TryParse(value, ignoreCase, out T res) ? res : null; + public static T try_convert(object value) { var numeric_type = Enum.GetUnderlyingType(typeof(T)); var numeric_value = Convert.ChangeType(value, numeric_type); diff --git a/src/client/spark_websocket/Core/WebsocketDataSync.cs b/src/client/spark_websocket/Core/WebsocketDataSync.cs index f1700356f..138b788a5 100644 --- a/src/client/spark_websocket/Core/WebsocketDataSync.cs +++ b/src/client/spark_websocket/Core/WebsocketDataSync.cs @@ -16,6 +16,7 @@ namespace bts.websocket_core { using bts.api.position; using bts.api.synthetic; using Spark.Api.Services; + using Bts.DataStreamCore; /// @@ -26,10 +27,12 @@ namespace bts.websocket_core { readonly AdminService adminService; readonly Spark.Api.IServiceProvider _service_ctx; readonly IUserDataPermissions userDataPermissions; + readonly IDataStreamFactory streamFactory; - public WebsocketDataSync(Spark.Api.IServiceProvider service_ctx, IUserDataPermissions userDataPermissions) { + public WebsocketDataSync(Spark.Api.IServiceProvider service_ctx, IUserDataPermissions userDataPermissions, IDataStreamFactory streamFactory) { _service_ctx = service_ctx; this.userDataPermissions = userDataPermissions; + this.streamFactory = streamFactory; adminService = service_ctx.GetService(); } @@ -328,6 +331,7 @@ namespace bts.websocket_core { req, Md_Client, Orders_Client, + streamFactory, obj => send_message(obj, req.client_id), out hdlr, out status)) { diff --git a/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs b/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs index 5d6794eab..7fab17507 100644 --- a/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs +++ b/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs @@ -224,6 +224,7 @@ namespace bts.websocket_core { } static bool TryGetDataStream( WebsocketRequest req, + IDataStreamFactory streamFactory, [NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(false)] out string error ) { @@ -253,7 +254,7 @@ namespace bts.websocket_core { } Exception? exception = null; try { - stream = exp.getView(new Dictionary(0)); + stream = exp.getView(new Dictionary(0), streamFactory); } catch (Exception ex) { exception = ex; } @@ -306,6 +307,7 @@ namespace bts.websocket_core { WebsocketRequest req, MarketDataClient md, OrdersClient oc, + IDataStreamFactory streamFactory, Action sm, [NotNullWhen(true)] out WebsocketSubscriptionHandler? hdlr, out string status @@ -372,7 +374,7 @@ namespace bts.websocket_core { } case SUBSCRIPTION_TYPE.DATA_STREAM: { // TODO: make this nullable - if (!TryGetDataStream(req, out var filter, out status)) { + if (!TryGetDataStream(req, streamFactory, out var filter, out status)) { return false; } sub = GetDataStreamSubscriber(filter, id.Value.Val);