diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamMetadata.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamMetadata.cs index 53d57791e..b8835f79b 100644 --- a/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamMetadata.cs +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamMetadata.cs @@ -15,13 +15,20 @@ public interface IDataStreamMetadata { public IEnumerable AlternateNames { get; } } -public class DataStreamMetadata : IDataStreamMetadata { +public interface IStructuredDataStreamMetadata : IDataStreamMetadata { + IEnumerable<(string? Name, Type? Type)[]> Signatures { get; } +} + +public class DataStreamMetadata : IDataStreamMetadata, IStructuredDataStreamMetadata { public string? Name { get; set; } public List? AlternateNames { get; set; } + public List<(string? Name, Type? Type)[]>? Signatures { get; set; } string IDataStreamMetadata.Name => Name ?? string.Empty; IEnumerable IDataStreamMetadata.AlternateNames => AlternateNames ?? new(0); + IEnumerable<(string? Name, Type? Type)[]> IStructuredDataStreamMetadata.Signatures => + Signatures ?? Enumerable.Empty<(string?, Type?)[]>(); public static DataStreamMetadata FromType(Type type) { var friendlyName = type.GetCustomAttribute()?.friendlyName; diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveTradeStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveTradeStream.cs index a38e5cce1..419fbcff9 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveTradeStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveTradeStream.cs @@ -5,7 +5,7 @@ using bts.api; - [DataStreamUserRespresentation("LiveTradeStream")] + [DataStreamUserRespresentation(false)] public class LiveTradeStream : ViewStream { public LiveTradeStream(string target) : this(SecurityType.Future, target) { } diff --git a/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs b/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs index f6a118f45..b74cd18a5 100644 --- a/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs +++ b/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.DependencyInjection; public static class SharedStreamsServices { public static IServiceCollection AddMarketDataStreams(this IServiceCollection services) => services.addStreamFactory() + .addStreamFactory() .addStreamFactory() .addStreamFactory() .addStreamFactory() diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/MarketDataArgStore.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/MarketDataArgStore.cs new file mode 100644 index 000000000..321816358 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/MarketDataArgStore.cs @@ -0,0 +1,198 @@ +namespace Bts.DataStreamCore.StreamFactories; + +using bts.api; + +using Microsoft.Extensions.Logging; + +using System.Collections.Immutable; +using System.Diagnostics.CodeAnalysis; + +class MarketDataArgStore { + ImmutableList<(string? Name, int? Index, object? Value)> argsBacking; + readonly ILogger? logger; + readonly string streamType; + StreamFactoryArgStore? argStoreBacking; + + public MarketDataArgStore(IEnumerable<(string? Name, int? Index, object? Value)> args, string? streamType = null, ILogger? logger = null) { + argsBacking = args.ToImmutableList(); + this.streamType = streamType ?? "anonStream"; + this.logger = logger; + } + + public StreamFactoryArgStore ArgStore => + argStoreBacking ?? new(args, streamType, logger); + + ImmutableList<(string? Name, int? Index, object? Value)> args { + get => argsBacking; + set { + argsBacking = value; + argStoreBacking = null; + } + } + + public Venue? GetVenue(bool required = true, bool removeArg = true) => + GetVenue(new[] { "venue", "exchange", "exch" }, required: required, removeArg: removeArg); + + public Venue? GetVenue(string[] preferredNames, bool required = true, bool removeArg = true) { + return GetValue(preferredNames, required, removeArg, out Venue result, TryConvert) + ? result + : null; + + static bool TryConvert(object? obj, out Venue result) { + if (obj is Venue res) { + result = res; + return true; + } + var str = obj as string ?? obj?.ToString(); + if (TryParseVenue(str, out res)) { + result = res; + return true; + } + result = Venue.Unset; + return false; + } + } + + public string? GetVenueString(bool required = true, bool removeArg = true) => + GetVenueString(new[] { "venue", "exchange", "exch" }, required: required, removeArg: removeArg); + + public string? GetVenueString(string[] preferredNames, bool required = true, bool removeArg = true) { + return GetValue(preferredNames, required: required, removeArg: removeArg, out string? result, TryConvert) + ? result + : null; + + static bool TryConvert(object? obj, [NotNullWhen(true)] out string? result) { + result = obj as string ?? obj?.ToString(); + result = result?.ToUpper()?.Replace("VENUE_", ""); + return result is not null; + } + } + + public SecurityType? GetSecurityType(bool required = true, bool removeArg = true) => + GetSecurityType(new[] { "secType", "securityType", "type" }, required: required, removeArg: removeArg); + + public SecurityType? GetSecurityType(string[] preferredNames, bool required = true, bool removeArg = true) { + return GetValue(preferredNames, required, removeArg, out SecurityType result, TryConvert) + ? result + : null; + + static bool TryConvert(object? obj, out SecurityType result) { + if (obj is SecurityType res) { + result = res; + return true; + } + var str = obj as string ?? obj?.ToString(); + if (TryParseSecurityType(str, out result)) { + return true; + } + result = SecurityType.Unset; + return false; + } + } + + public bool GetDateTime([NotNullWhen(true)] out DateTime? result, out int index, bool removeArg = true) { + bool success = ArgStore.GetDateTime(out result, out index, false); + if (success && removeArg) { + args = args.RemoveAt(index); + } + return success; + } + + public bool GetSymbol([NotNullWhen(true)] out string? result, bool required = true) { + result = GetSymbol(required); + return result is not null; + } + + public string? GetSymbol(bool required = true) { + var rv = ArgStore.Get("symbol", "product", "productCode", "asset")?.ToString(); + if (required && logger is not null && string.IsNullOrEmpty(rv)) { + logger.LogMissingParam(streamType, "symbol"); + } + return rv; + } + + public bool GetSecurityName([NotNullWhen(true)] out string? result, bool required = true) { + result = GetSecurityName(required); + return result is not null; + } + + public string? GetSecurityName(bool required = true, bool removeArg = true) { + if (ArgStore.TryGet(new[] { "securityName", "name", "sec", "secName", "security" }, out var arg, out int index)) { + if (removeArg) { + args = args.RemoveAt(index); + } + return arg.Value?.ToString(); + } + if (required && logger is not null) { + logger.LogMissingParam(streamType, "securityName"); + } + return null; + } + + public bool GetValue(string[] preferredNames, bool required, bool removeArg, [MaybeNullWhen(false)] out T result, StreamFactoryArgStore.TryConvert tryConvert) { + foreach (var name in preferredNames) { + if (ArgStore.TryGet(new[] { name }, out var arg, out int index)) { + if (removeArg) { + args = args.RemoveAt(index); + } + if (tryConvert(arg.Value, out result)) { + return true; + } + if (required) { + logger.LogInvalidParams(streamType, $"Failed to convert '{name}' param to {typeof(T)}"); + } + return false; + } + } + int i = 0; + foreach (var (name, pos, value) in args) { + if (tryConvert(value, out result)) { + if (removeArg) { + args = args.RemoveAt(i); + } + return true; + } + i++; + } + if (required) { + logger.LogMissingParam(streamType, preferredNames.FirstOrDefault() ?? "venue"); + } + result = default; + return false; + } + + public static Venue ParseVenue(string? str) => + TryParseVenue(str, out var venue) + ? venue + : throw new ArgumentException("Failed to parse venue", nameof(str)); + + public static bool TryParseVenue(string? str, out Venue v) { + if (str is null) { + v = Venue.Unset; + return false; + } + if (Enum.TryParse(str, true, out v) || Enum.TryParse("VENUE_" + str, true, out v)) { + return true; + } + v = str.ToUpper() switch { + "CME" or "XCME" or "CEC" or "XCEC" => Venue.Glbx, + "CBOT" or "CBT" or "XCBT" => Venue.Cboe, + _ => Venue.Unset, + }; + return v != Venue.Unset; + } + + public static SecurityType ParseSecurityType(string? str) => + TryParseSecurityType(str, out var secType) + ? secType + : throw new ArgumentException("Failed to parse sec type", nameof(str)); + + public static bool TryParseSecurityType(string? str, out SecurityType t) { + if (str is null) { + // bbo(symbol: "ZN", venue: "GLBX", begin: "2024-10-03T18:00:00-00:05", end: "2024-10-03T19:00:00-00:05", interval: "PT1S") + t = SecurityType.Unset; + return false; + } + return Enum.TryParse(str, true, out t) || Enum.TryParse("SECURITYTYPE_" + str, true, out t); + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/TradeSchemaBuilder.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/TradeSchemaBuilder.cs new file mode 100644 index 000000000..59d33851b --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/TradeSchemaBuilder.cs @@ -0,0 +1,92 @@ +namespace Bts.SharedStreams.StreamFactories; + +using BCG; + +using bts.api; + +using Bts.DataStreamCore; + +using mbts.spark; +using mbts.utils; + +public class TradeSchemaBuilder { + public bool ShowNotional { get; set; } + public bool ShowQuantity { get; set; } = true; + public bool ShowPrice { get; set; } = true; + public bool ShowVenue { get; set; } + public bool ShowAsset { get; set; } = true; + public bool ShowSecurityType { get; set; } + public bool ShowSecurityName { get; set; } + public bool ShowSecurityId { get; set; } = true; + public bool ShowCashPerPoint { get; set; } + public bool ShowTimestamp { get; set; } = true; + public string TableName { get; set; } = "Trade"; + + public IBlueTableSchema ToSchema() { + var bd = new BlueTableSchemaBuilder(TableName) { + ("id", typeof(int)) + }; + if (ShowTimestamp) { + bd.Add(WellKnownColumns.Timestamp); + } + if (ShowVenue) { + bd.Add(WellKnownColumns.Venue); + } + if (ShowSecurityType) { + bd.Add(WellKnownColumns.SecurityType); + } + if (ShowAsset) { + bd.Add(WellKnownColumns.Asset); + } + if (ShowSecurityName) { + bd.Add(WellKnownColumns.SecurityName); + } + if (ShowSecurityId) { + bd.Add(WellKnownColumns.SecurityId); + } + if (ShowCashPerPoint) { + bd.Add(WellKnownColumns.CashPerPoint); + } + if (ShowPrice) { + bd.Add(WellKnownColumns.Price); + } + if (ShowQuantity) { + bd.Add(WellKnownColumns.Quantity); + } + return bd.ToSchema(); + } + + public BlueTableSchemaAdapter TradeInfoAdapter() { + var adapter = new BlueTableSchemaAdapter(TableName); + adapter.AddColumn("id", r => r.RecordId); + if (ShowTimestamp) { + adapter.AddColumn(WellKnownColumns.Timestamp.Name, b => new LLDateTime((ulong)b.trade.Time).to_datetime()); + } + if (ShowVenue) { + adapter.AddColumn(WellKnownColumns.Venue.Name, b => b.security.Exchange.ToString()); + } + if (ShowSecurityType) { + adapter.AddColumn(WellKnownColumns.SecurityType.Name, b => b.security.security_type().ToString()); + } + if (ShowAsset) { + adapter.AddColumn(WellKnownColumns.Asset.Name, b => b.security.Symbol); + } + if (ShowSecurityName) { + adapter.AddColumn(WellKnownColumns.SecurityName.Name, b => b.security.ToString()); + } + if (ShowSecurityId) { + adapter.AddColumn(WellKnownColumns.SecurityId.Name, b => b.security.Id); + } + if (ShowCashPerPoint) { + adapter.AddColumn(WellKnownColumns.CashPerPoint.Name, b => b.security.ContractSpec.CashPerPoint); + } + if (ShowPrice) { + adapter.AddColumn(WellKnownColumns.Price.Name, b => b.trade.Mantissa * Math.Pow(10, b.trade.Exponent)); + } + if (ShowQuantity) { + // TODO: Make trade size a float (it's still uint) + adapter.AddColumn(WellKnownColumns.Quantity.Name, b => b.trade.Size); + } + return adapter; + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/TradeStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/TradeStreamFactory.cs new file mode 100644 index 000000000..715f64f16 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/TradeStreamFactory.cs @@ -0,0 +1,153 @@ +namespace Bts.SharedStreams.StreamFactories; + +using BCG; + +using bts.collections; + +using Bts.DataStream; +using Bts.DataStreamCore; +using Bts.DataStreamCore.StreamFactories; + +using mbts.spark; +using mbts.utils; + +using Microsoft.Extensions.Logging; + +using Spark.Api.MarketData; + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Reactive.Linq; + +public class TradeStreamFactory : DataStreamFactory, IRecursiveDataStreamFactory { + readonly LiveTradeStreamFactory liveTradeFactory; + IDataStreamFactory? compositeFactory = null; + + public TradeStreamFactory(IMarketDataClient? mdClient, ILogger logger) { + this.Add(new[] { "trade" }, CreateTrade, logger); + liveTradeFactory = new LiveTradeStreamFactory(mdClient, logger); + Add(liveTradeFactory); + + ViewStream? CreateTrade(TradeStreamParams streamParams) { + if (mdClient is null) { + return null; + } + var secCriteria = streamParams.GetSecurityCriteria(); + var schemaBd = new TradeSchemaBuilder(); + return TradeStream(mdClient.Trades(secCriteria), schemaBd.ToSchema(), schemaBd.TradeInfoAdapter()) + ?.AsViewStream(); + } + } + + public void OnCompositeFactory(IDataStreamFactory compositeFactory) { + this.compositeFactory = compositeFactory; + liveTradeFactory.OnCompositeFactory(compositeFactory); + } + + public static IObservable TradeStream(IObservable> trades, IBlueTableSchema tradeSchema, BlueTableSchemaAdapter tradeAdapter) => + trades.Scan((IDataPacket)null!, (dp, trades) => { + var rowList = trades.WrapSelector(t => (IBlueRow)new BlueRowAdapter(tradeAdapter, t), null); + var lastTable = dp?.newRows as BlueTableImmutable ?? new BlueTableImmutable(tradeSchema); + var bd = lastTable.ToDataPacketBuilder(); + foreach (var t in trades) { + bd.AddOrUpdate(new BlueRowAdapter(tradeAdapter, t)); + } + return bd.ToDataPacket(); + }); + + static IObservable tradeStream(IMarketDataClient mdClient, TradeStreamParams streamParams) { + var schemaBd = streamParams.GetSchemaBuilder(); + return TradeStream(mdClient.Trades(streamParams.GetSecurityCriteria()), schemaBd.ToSchema(), schemaBd.TradeInfoAdapter()); + } + + class LiveTradeStreamFactory : IDataStreamFactory, IRecursiveDataStreamFactory { + static readonly IDataStreamMetadata metadata = new DataStreamMetadata() { + Name = "LiveTradeStream", + AlternateNames = new(), + Signatures = new() { + new (string?, Type?)[] { + ("symbol", typeof(string)) + }, + new (string?, Type?)[] { + ("secType", typeof(string)), + ("symbol", typeof(string)), + } + } + }; + readonly IMarketDataClient? mdClient; + readonly ILogger logger; + IDataStreamFactory? compositeFactory = null; + + public LiveTradeStreamFactory(IMarketDataClient? mdClient, ILogger logger) { + this.mdClient = mdClient; + this.logger = logger; + } + + public void OnCompositeFactory(IDataStreamFactory compositeFactory) => + this.compositeFactory = compositeFactory; + + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (!metadata.IsMatch(streamId)) { + return null; + } + var argList = args.ToList(); + switch (argList.Count) { + case 1: { + var sym = argList[0].Value?.ToString() ?? throw new ArgumentException("Symbol arg is null or empty", nameof(args)); + if (sym == "ES" && compositeFactory is not null) { + return compositeFactory.Create("ReplayFutureLiveTradeStream", new (string?, int?, object?)[] { ("contract", 0, sym) }); + } + if (!checkHasLiveMd()) { + return null; + } + var streamParams = new TradeStreamParams() { + Symbol = sym + }; + return tradeStream(mdClient, streamParams).AsViewStream(); + } + case 2: { + if (!checkHasLiveMd()) { + return null; + } + var secTypeStr = argList[0].Value?.ToString() ?? string.Empty; + if (!MarketDataArgStore.TryParseSecurityType(secTypeStr, out var secType)) { + throw new ArgumentException("Failed to parse SecurityType arg", nameof(args)); + } + var sym = argList[1].Value?.ToString() ?? throw new ArgumentException("Symbol arg is null or empty", nameof(args)); + if (sym == "ES" && secType == bts.api.SecurityType.Future && compositeFactory is not null) { + return compositeFactory.Create("ReplayFutureLiveBBOStream", new (string?, int?, object?)[] { ("contract", 0, sym) }); + } + var streamParams = new TradeStreamParams() { + SecurityType = secType, + Symbol = sym, + }; + return tradeStream(mdClient, streamParams).AsViewStream(); + } + default: + return null; + } + } + + public IEnumerable GetSupportedStreams() { + yield return metadata; + } + + [MemberNotNullWhen(true, nameof(mdClient))] + bool checkHasLiveMd() { + if (mdClient is not null) { + return true; + } + logger.LogError("Market data is unavailable (md client is null)"); + return false; + } + } + + public class LiveBboStreamParams { + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex }, Position = 0, IsRequired = false)] + public string? SecurityType { get; set; } + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SymbolRegex }, Position = 1, IsRequired = false)] + public string? ContinuousContract { get; set; } + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/TradeStreamParams.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/TradeStreamParams.cs new file mode 100644 index 000000000..810ab21e7 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/TradeStreamParams.cs @@ -0,0 +1,53 @@ +namespace Bts.SharedStreams.StreamFactories; + +using bts.api; + +using Bts.DataStreamCore; + +public class TradeStreamParams { + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.VenueRegex })] + public Venue? Venue { get; set; } + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SymbolRegex })] + public string? Symbol { get; set; } + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex })] + public SecurityType? SecurityType { get; set; } + + [StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityNameRegex })] + public string? SecurityName { get; set; } + + [StreamArgument(RegexNames = new[] { "$(show|include)?(price|px|p)^" })] + public bool ShowPrice { get; set; } = true; + + [StreamArgument(RegexNames = new[] { "$(show|include)?(quantity|qty|q)^" })] + public bool ShowQuantity { get; set; } = true; + + [StreamArgument(RegexNames = new[] { "$(show|include)?(notional|notl)^" })] + public bool ShowNotional { get; set; } + + [StreamArgument(RegexNames = new[] { "$(show|include)?(cashPerPoint|cpp)^" })] + public bool ShowCashPerPoint { get; set; } = true; + + public mbts.spark.SecurityCriteria GetSecurityCriteria() { + var clause = new mbts.spark.SecurityCriteriaClause( + exchanges: Venue.HasValue ? new[] { Venue.Value } : null, + security_types: SecurityType.HasValue ? new[] { SecurityType.Value } : null, + symbols: !string.IsNullOrEmpty(Symbol) ? new[] { Symbol } : null, + names: !string.IsNullOrEmpty(SecurityName) ? new[] { SecurityName } : null); + return clause.is_defined ? new(new[] { clause }) : new(); + } + + public TradeSchemaBuilder GetSchemaBuilder() => + new() { + ShowVenue = true, + ShowSecurityType = true, + ShowAsset = true, + ShowSecurityName = true, + ShowSecurityId = true, + ShowPrice = ShowPrice, + ShowQuantity = ShowQuantity, + ShowNotional = ShowNotional, + ShowCashPerPoint = ShowCashPerPoint, + }; +}