refactor: Launch LiveTradeStream via IDataStreamFactory

This commit is contained in:
John Colvin committed 2025-11-19 14:27:48 -05:00
1 parent c21cd2ba6d
commit d7300cbddb
7 files changed
+506 -2

No files matched your search

@@ -15,13 +15,20 @@ public interface IDataStreamMetadata {
public IEnumerable<string> AlternateNames { get; }
}
public class DataStreamMetadata : IDataStreamMetadata {
public interface IStructuredDataStreamMetadata : IDataStreamMetadata {
IEnumerable<(string? Name, Type? Type)[]> Signatures { get; }
}
public class DataStreamMetadata : IDataStreamMetadata, IStructuredDataStreamMetadata {
public string? Name { get; set; }
public List<string>? AlternateNames { get; set; }
public List<(string? Name, Type? Type)[]>? Signatures { get; set; }
string IDataStreamMetadata.Name => Name ?? string.Empty;
IEnumerable<string> IDataStreamMetadata.AlternateNames =>
AlternateNames ?? new(0);
IEnumerable<(string? Name, Type? Type)[]> IStructuredDataStreamMetadata.Signatures =>
Signatures ?? Enumerable.Empty<(string?, Type?)[]>();
public static DataStreamMetadata FromType(Type type) {
var friendlyName = type.GetCustomAttribute<DataStreamUserRespresentation>()?.friendlyName;
@@ -5,7 +5,7 @@
using bts.api;
[DataStreamUserRespresentation("LiveTradeStream")]
[DataStreamUserRespresentation(false)]
public class LiveTradeStream : ViewStream {
public LiveTradeStream(string target)
: this(SecurityType.Future, target) { }
@@ -11,6 +11,7 @@ using Microsoft.Extensions.DependencyInjection;
public static class SharedStreamsServices {
public static IServiceCollection AddMarketDataStreams(this IServiceCollection services) =>
services.addStreamFactory<BookStreamFactory>()
.addStreamFactory<TradeStreamFactory>()
.addStreamFactory<SpreadBookStreamFactory>()
.addStreamFactory<InstrumentStreamFactory>()
.addStreamFactory<SpatialFuturePairsStreamFactory>()
@@ -0,0 +1,198 @@
namespace Bts.DataStreamCore.StreamFactories;
using bts.api;
using Microsoft.Extensions.Logging;
using System.Collections.Immutable;
using System.Diagnostics.CodeAnalysis;
class MarketDataArgStore {
ImmutableList<(string? Name, int? Index, object? Value)> argsBacking;
readonly ILogger? logger;
readonly string streamType;
StreamFactoryArgStore? argStoreBacking;
public MarketDataArgStore(IEnumerable<(string? Name, int? Index, object? Value)> args, string? streamType = null, ILogger? logger = null) {
argsBacking = args.ToImmutableList();
this.streamType = streamType ?? "anonStream";
this.logger = logger;
}
public StreamFactoryArgStore ArgStore =>
argStoreBacking ?? new(args, streamType, logger);
ImmutableList<(string? Name, int? Index, object? Value)> args {
get => argsBacking;
set {
argsBacking = value;
argStoreBacking = null;
}
}
public Venue? GetVenue(bool required = true, bool removeArg = true) =>
GetVenue(new[] { "venue", "exchange", "exch" }, required: required, removeArg: removeArg);
public Venue? GetVenue(string[] preferredNames, bool required = true, bool removeArg = true) {
return GetValue(preferredNames, required, removeArg, out Venue result, TryConvert)
? result
: null;
static bool TryConvert(object? obj, out Venue result) {
if (obj is Venue res) {
result = res;
return true;
}
var str = obj as string ?? obj?.ToString();
if (TryParseVenue(str, out res)) {
result = res;
return true;
}
result = Venue.Unset;
return false;
}
}
public string? GetVenueString(bool required = true, bool removeArg = true) =>
GetVenueString(new[] { "venue", "exchange", "exch" }, required: required, removeArg: removeArg);
public string? GetVenueString(string[] preferredNames, bool required = true, bool removeArg = true) {
return GetValue(preferredNames, required: required, removeArg: removeArg, out string? result, TryConvert)
? result
: null;
static bool TryConvert(object? obj, [NotNullWhen(true)] out string? result) {
result = obj as string ?? obj?.ToString();
result = result?.ToUpper()?.Replace("VENUE_", "");
return result is not null;
}
}
public SecurityType? GetSecurityType(bool required = true, bool removeArg = true) =>
GetSecurityType(new[] { "secType", "securityType", "type" }, required: required, removeArg: removeArg);
public SecurityType? GetSecurityType(string[] preferredNames, bool required = true, bool removeArg = true) {
return GetValue(preferredNames, required, removeArg, out SecurityType result, TryConvert)
? result
: null;
static bool TryConvert(object? obj, out SecurityType result) {
if (obj is SecurityType res) {
result = res;
return true;
}
var str = obj as string ?? obj?.ToString();
if (TryParseSecurityType(str, out result)) {
return true;
}
result = SecurityType.Unset;
return false;
}
}
public bool GetDateTime([NotNullWhen(true)] out DateTime? result, out int index, bool removeArg = true) {
bool success = ArgStore.GetDateTime(out result, out index, false);
if (success && removeArg) {
args = args.RemoveAt(index);
}
return success;
}
public bool GetSymbol([NotNullWhen(true)] out string? result, bool required = true) {
result = GetSymbol(required);
return result is not null;
}
public string? GetSymbol(bool required = true) {
var rv = ArgStore.Get("symbol", "product", "productCode", "asset")?.ToString();
if (required && logger is not null && string.IsNullOrEmpty(rv)) {
logger.LogMissingParam(streamType, "symbol");
}
return rv;
}
public bool GetSecurityName([NotNullWhen(true)] out string? result, bool required = true) {
result = GetSecurityName(required);
return result is not null;
}
public string? GetSecurityName(bool required = true, bool removeArg = true) {
if (ArgStore.TryGet(new[] { "securityName", "name", "sec", "secName", "security" }, out var arg, out int index)) {
if (removeArg) {
args = args.RemoveAt(index);
}
return arg.Value?.ToString();
}
if (required && logger is not null) {
logger.LogMissingParam(streamType, "securityName");
}
return null;
}
public bool GetValue<T>(string[] preferredNames, bool required, bool removeArg, [MaybeNullWhen(false)] out T result, StreamFactoryArgStore.TryConvert<T> tryConvert) {
foreach (var name in preferredNames) {
if (ArgStore.TryGet(new[] { name }, out var arg, out int index)) {
if (removeArg) {
args = args.RemoveAt(index);
}
if (tryConvert(arg.Value, out result)) {
return true;
}
if (required) {
logger.LogInvalidParams(streamType, $"Failed to convert '{name}' param to {typeof(T)}");
}
return false;
}
}
int i = 0;
foreach (var (name, pos, value) in args) {
if (tryConvert(value, out result)) {
if (removeArg) {
args = args.RemoveAt(i);
}
return true;
}
i++;
}
if (required) {
logger.LogMissingParam(streamType, preferredNames.FirstOrDefault() ?? "venue");
}
result = default;
return false;
}
public static Venue ParseVenue(string? str) =>
TryParseVenue(str, out var venue)
? venue
: throw new ArgumentException("Failed to parse venue", nameof(str));
public static bool TryParseVenue(string? str, out Venue v) {
if (str is null) {
v = Venue.Unset;
return false;
}
if (Enum.TryParse(str, true, out v) || Enum.TryParse("VENUE_" + str, true, out v)) {
return true;
}
v = str.ToUpper() switch {
"CME" or "XCME" or "CEC" or "XCEC" => Venue.Glbx,
"CBOT" or "CBT" or "XCBT" => Venue.Cboe,
_ => Venue.Unset,
};
return v != Venue.Unset;
}
public static SecurityType ParseSecurityType(string? str) =>
TryParseSecurityType(str, out var secType)
? secType
: throw new ArgumentException("Failed to parse sec type", nameof(str));
public static bool TryParseSecurityType(string? str, out SecurityType t) {
if (str is null) {
// bbo(symbol: "ZN", venue: "GLBX", begin: "2024-10-03T18:00:00-00:05", end: "2024-10-03T19:00:00-00:05", interval: "PT1S")
t = SecurityType.Unset;
return false;
}
return Enum.TryParse(str, true, out t) || Enum.TryParse("SECURITYTYPE_" + str, true, out t);
}
}
@@ -0,0 +1,92 @@
namespace Bts.SharedStreams.StreamFactories;
using BCG;
using bts.api;
using Bts.DataStreamCore;
using mbts.spark;
using mbts.utils;
public class TradeSchemaBuilder {
public bool ShowNotional { get; set; }
public bool ShowQuantity { get; set; } = true;
public bool ShowPrice { get; set; } = true;
public bool ShowVenue { get; set; }
public bool ShowAsset { get; set; } = true;
public bool ShowSecurityType { get; set; }
public bool ShowSecurityName { get; set; }
public bool ShowSecurityId { get; set; } = true;
public bool ShowCashPerPoint { get; set; }
public bool ShowTimestamp { get; set; } = true;
public string TableName { get; set; } = "Trade";
public IBlueTableSchema ToSchema() {
var bd = new BlueTableSchemaBuilder(TableName) {
("id", typeof(int))
};
if (ShowTimestamp) {
bd.Add(WellKnownColumns.Timestamp);
}
if (ShowVenue) {
bd.Add(WellKnownColumns.Venue);
}
if (ShowSecurityType) {
bd.Add(WellKnownColumns.SecurityType);
}
if (ShowAsset) {
bd.Add(WellKnownColumns.Asset);
}
if (ShowSecurityName) {
bd.Add(WellKnownColumns.SecurityName);
}
if (ShowSecurityId) {
bd.Add(WellKnownColumns.SecurityId);
}
if (ShowCashPerPoint) {
bd.Add(WellKnownColumns.CashPerPoint);
}
if (ShowPrice) {
bd.Add(WellKnownColumns.Price);
}
if (ShowQuantity) {
bd.Add(WellKnownColumns.Quantity);
}
return bd.ToSchema();
}
public BlueTableSchemaAdapter<TradeInfo> TradeInfoAdapter() {
var adapter = new BlueTableSchemaAdapter<TradeInfo>(TableName);
adapter.AddColumn("id", r => r.RecordId);
if (ShowTimestamp) {
adapter.AddColumn(WellKnownColumns.Timestamp.Name, b => new LLDateTime((ulong)b.trade.Time).to_datetime());
}
if (ShowVenue) {
adapter.AddColumn(WellKnownColumns.Venue.Name, b => b.security.Exchange.ToString());
}
if (ShowSecurityType) {
adapter.AddColumn(WellKnownColumns.SecurityType.Name, b => b.security.security_type().ToString());
}
if (ShowAsset) {
adapter.AddColumn(WellKnownColumns.Asset.Name, b => b.security.Symbol);
}
if (ShowSecurityName) {
adapter.AddColumn(WellKnownColumns.SecurityName.Name, b => b.security.ToString());
}
if (ShowSecurityId) {
adapter.AddColumn(WellKnownColumns.SecurityId.Name, b => b.security.Id);
}
if (ShowCashPerPoint) {
adapter.AddColumn(WellKnownColumns.CashPerPoint.Name, b => b.security.ContractSpec.CashPerPoint);
}
if (ShowPrice) {
adapter.AddColumn(WellKnownColumns.Price.Name, b => b.trade.Mantissa * Math.Pow(10, b.trade.Exponent));
}
if (ShowQuantity) {
// TODO: Make trade size a float (it's still uint)
adapter.AddColumn(WellKnownColumns.Quantity.Name, b => b.trade.Size);
}
return adapter;
}
}
@@ -0,0 +1,153 @@
namespace Bts.SharedStreams.StreamFactories;
using BCG;
using bts.collections;
using Bts.DataStream;
using Bts.DataStreamCore;
using Bts.DataStreamCore.StreamFactories;
using mbts.spark;
using mbts.utils;
using Microsoft.Extensions.Logging;
using Spark.Api.MarketData;
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Reactive.Linq;
public class TradeStreamFactory : DataStreamFactory, IRecursiveDataStreamFactory {
readonly LiveTradeStreamFactory liveTradeFactory;
IDataStreamFactory? compositeFactory = null;
public TradeStreamFactory(IMarketDataClient? mdClient, ILogger<TradeStreamFactory> logger) {
this.Add<TradeStreamParams>(new[] { "trade" }, CreateTrade, logger);
liveTradeFactory = new LiveTradeStreamFactory(mdClient, logger);
Add(liveTradeFactory);
ViewStream? CreateTrade(TradeStreamParams streamParams) {
if (mdClient is null) {
return null;
}
var secCriteria = streamParams.GetSecurityCriteria();
var schemaBd = new TradeSchemaBuilder();
return TradeStream(mdClient.Trades(secCriteria), schemaBd.ToSchema(), schemaBd.TradeInfoAdapter())
?.AsViewStream();
}
}
public void OnCompositeFactory(IDataStreamFactory compositeFactory) {
this.compositeFactory = compositeFactory;
liveTradeFactory.OnCompositeFactory(compositeFactory);
}
public static IObservable<IDataPacket> TradeStream(IObservable<IList<TradeInfo>> trades, IBlueTableSchema tradeSchema, BlueTableSchemaAdapter<TradeInfo> tradeAdapter) =>
trades.Scan((IDataPacket)null!, (dp, trades) => {
var rowList = trades.WrapSelector(t => (IBlueRow)new BlueRowAdapter<TradeInfo>(tradeAdapter, t), null);
var lastTable = dp?.newRows as BlueTableImmutable ?? new BlueTableImmutable(tradeSchema);
var bd = lastTable.ToDataPacketBuilder();
foreach (var t in trades) {
bd.AddOrUpdate(new BlueRowAdapter<TradeInfo>(tradeAdapter, t));
}
return bd.ToDataPacket();
});
static IObservable<IDataPacket> tradeStream(IMarketDataClient mdClient, TradeStreamParams streamParams) {
var schemaBd = streamParams.GetSchemaBuilder();
return TradeStream(mdClient.Trades(streamParams.GetSecurityCriteria()), schemaBd.ToSchema(), schemaBd.TradeInfoAdapter());
}
class LiveTradeStreamFactory : IDataStreamFactory, IRecursiveDataStreamFactory {
static readonly IDataStreamMetadata metadata = new DataStreamMetadata() {
Name = "LiveTradeStream",
AlternateNames = new(),
Signatures = new() {
new (string?, Type?)[] {
("symbol", typeof(string))
},
new (string?, Type?)[] {
("secType", typeof(string)),
("symbol", typeof(string)),
}
}
};
readonly IMarketDataClient? mdClient;
readonly ILogger logger;
IDataStreamFactory? compositeFactory = null;
public LiveTradeStreamFactory(IMarketDataClient? mdClient, ILogger logger) {
this.mdClient = mdClient;
this.logger = logger;
}
public void OnCompositeFactory(IDataStreamFactory compositeFactory) =>
this.compositeFactory = compositeFactory;
public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) {
if (!metadata.IsMatch(streamId)) {
return null;
}
var argList = args.ToList();
switch (argList.Count) {
case 1: {
var sym = argList[0].Value?.ToString() ?? throw new ArgumentException("Symbol arg is null or empty", nameof(args));
if (sym == "ES" && compositeFactory is not null) {
return compositeFactory.Create("ReplayFutureLiveTradeStream", new (string?, int?, object?)[] { ("contract", 0, sym) });
}
if (!checkHasLiveMd()) {
return null;
}
var streamParams = new TradeStreamParams() {
Symbol = sym
};
return tradeStream(mdClient, streamParams).AsViewStream();
}
case 2: {
if (!checkHasLiveMd()) {
return null;
}
var secTypeStr = argList[0].Value?.ToString() ?? string.Empty;
if (!MarketDataArgStore.TryParseSecurityType(secTypeStr, out var secType)) {
throw new ArgumentException("Failed to parse SecurityType arg", nameof(args));
}
var sym = argList[1].Value?.ToString() ?? throw new ArgumentException("Symbol arg is null or empty", nameof(args));
if (sym == "ES" && secType == bts.api.SecurityType.Future && compositeFactory is not null) {
return compositeFactory.Create("ReplayFutureLiveBBOStream", new (string?, int?, object?)[] { ("contract", 0, sym) });
}
var streamParams = new TradeStreamParams() {
SecurityType = secType,
Symbol = sym,
};
return tradeStream(mdClient, streamParams).AsViewStream();
}
default:
return null;
}
}
public IEnumerable<IDataStreamMetadata> GetSupportedStreams() {
yield return metadata;
}
[MemberNotNullWhen(true, nameof(mdClient))]
bool checkHasLiveMd() {
if (mdClient is not null) {
return true;
}
logger.LogError("Market data is unavailable (md client is null)");
return false;
}
}
public class LiveBboStreamParams {
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex }, Position = 0, IsRequired = false)]
public string? SecurityType { get; set; }
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SymbolRegex }, Position = 1, IsRequired = false)]
public string? ContinuousContract { get; set; }
}
}
@@ -0,0 +1,53 @@
namespace Bts.SharedStreams.StreamFactories;
using bts.api;
using Bts.DataStreamCore;
public class TradeStreamParams {
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.VenueRegex })]
public Venue? Venue { get; set; }
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SymbolRegex })]
public string? Symbol { get; set; }
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityTypeRegex })]
public SecurityType? SecurityType { get; set; }
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityNameRegex })]
public string? SecurityName { get; set; }
[StreamArgument(RegexNames = new[] { "$(show|include)?(price|px|p)^" })]
public bool ShowPrice { get; set; } = true;
[StreamArgument(RegexNames = new[] { "$(show|include)?(quantity|qty|q)^" })]
public bool ShowQuantity { get; set; } = true;
[StreamArgument(RegexNames = new[] { "$(show|include)?(notional|notl)^" })]
public bool ShowNotional { get; set; }
[StreamArgument(RegexNames = new[] { "$(show|include)?(cashPerPoint|cpp)^" })]
public bool ShowCashPerPoint { get; set; } = true;
public mbts.spark.SecurityCriteria GetSecurityCriteria() {
var clause = new mbts.spark.SecurityCriteriaClause(
exchanges: Venue.HasValue ? new[] { Venue.Value } : null,
security_types: SecurityType.HasValue ? new[] { SecurityType.Value } : null,
symbols: !string.IsNullOrEmpty(Symbol) ? new[] { Symbol } : null,
names: !string.IsNullOrEmpty(SecurityName) ? new[] { SecurityName } : null);
return clause.is_defined ? new(new[] { clause }) : new();
}
public TradeSchemaBuilder GetSchemaBuilder() =>
new() {
ShowVenue = true,
ShowSecurityType = true,
ShowAsset = true,
ShowSecurityName = true,
ShowSecurityId = true,
ShowPrice = ShowPrice,
ShowQuantity = ShowQuantity,
ShowNotional = ShowNotional,
ShowCashPerPoint = ShowCashPerPoint,
};
}