refactor: Load IDataStreamFactory(s) with dependency injection

So we can create ViewStreams that depend on other services, config, etc.
Previously, the PrimitiveManager relied purely on static construction of
types through reflection.
This commit is contained in:
John Colvin committed 2025-11-19 13:53:18 -05:00
1 parent 7566b2560f
commit ea0a1fbcd3
59 files changed
+2124 -161

No files matched your search

+6
View File
@@ -25,6 +25,12 @@
"commands": [
"fake"
]
},
"dotnet-t4": {
"version": "3.0.0",
"commands": [
"t4"
]
}
}
}
@@ -3,23 +3,24 @@ namespace bts.BlueOrb {
using BCG;
using System.Collections.Generic;
using Bts.DataStreamExpressions;
using Bts.DataStreamCore;
// All this does is take in a dsl expression, parse it and fire the data that come out of running it.
// So for example, if an expresison is passed
[DataStreamUserRespresentation("dsl")]
public class DSLStream : ViewStream {
private readonly string _dsl;
public DSLStream(string dsl) {
public DSLStream(string dsl, IDataStreamFactory streamFactory) {
Log.Gen.info($"DSLStream.cons dsl: {dsl}");
_dsl = dsl;
ViewStream rs = build_stream(dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
var rs = build_stream(streamFactory, dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
mySetRealSource(rs);
}
static ViewStream? build_stream(string stream_expression) {
static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) {
try {
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
return rv;
} catch (Exception ex) {
@@ -35,12 +35,12 @@ namespace BCG {
view_publisher = view_publisher_;
}
}
static ViewStream? build_stream(string stream_expression)
static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression)
{
try
{
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
return rv;
} catch (Exception ex) {
@@ -52,15 +52,17 @@ namespace BCG {
_stream_pub_map = new Dictionary<string, StreamPubUnit>();
readonly IPubTableBroker _broker;
readonly IManagedSubscriber _sub;
readonly IDataStreamFactory streamFactory;
readonly object syncRoot = new();
IDisposable? cloudStreamsSub;
IDisposable? remoteRequestsSub;
bool isDisposed;
public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker, IDataStreamFactory streamFactory) {
_sub = managedSubscriber;
_broker = pubTableBroker;
this.streamFactory = streamFactory;
}
public Task StartAsync(CancellationToken cancellationToken) {
@@ -103,7 +105,11 @@ namespace BCG {
private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
ViewStream stream = build_stream(stream_expression);
var stream = build_stream(streamFactory, stream_expression);
if (stream is null) {
Log.DataFlow.error($"Failed to spawn remote stream. name: {name}; expression: {stream_expression}");
return;
}
_expose_to_ws(name, stream);
if (persist) {
string tempDir = Path.GetTempPath();
@@ -19,6 +19,8 @@ using Newtonsoft.Json;
using BCG;
using System.Runtime.CompilerServices;
using bts;
using Spark.Core;
using System.Numerics;
namespace bts {
@@ -91,13 +93,13 @@ namespace BCG {
return BlueDataUtility.get_row_datetime_value(r, col_no, err_value);
}
public static double? GetDouble(this IBlueRow r, int colNo) =>
GetDouble(r[colNo]);
public static double? GetFloat64(this IBlueRow r, int colNo) =>
GetFloat64(r[colNo]);
public static double? GetDouble(this IBlueRow r, string colName) =>
GetDouble(r[colName]);
public static double? GetFloat64(this IBlueRow r, string colName) =>
GetFloat64(r[colName]);
public static double? GetDouble(object obj) {
public static double? GetFloat64(object obj) {
if (obj is double d) {
return d;
}
@@ -125,6 +127,127 @@ namespace BCG {
return null;
}
public static int? GetInt32(this IBlueRow r, int colNo) =>
GetInt32(r[colNo]);
public static int? GetInt32(this IBlueRow r, string colName) =>
GetInt32(r[colName]);
public static int? GetInt32(object obj) {
checked {
if (obj is double d) {
return (int)d;
}
if (obj is int i32) {
return i32;
}
if (obj is long i64) {
return (int)i64;
}
if (obj is string s) {
return double.TryParse(s, out d) ? (int)d : null;
}
if (obj is uint u32) {
return (int)u32;
}
if (obj is ulong u64) {
return (int)u64;
}
if (obj is short i16) {
return i16;
}
if (obj is ushort u16) {
return u16;
}
}
return null;
}
public static long? GetInt64(this IBlueRow r, int colNo) =>
GetInt64(r[colNo]);
public static long? GetInt64(this IBlueRow r, string colName) =>
GetInt64(r[colName]);
public static long? GetInt64(object obj) {
checked {
if (obj is double d) {
return (long)d;
}
if (obj is int i32) {
return i32;
}
if (obj is long i64) {
return i64;
}
if (obj is string s) {
return double.TryParse(s, out d) ? (long)d : null;
}
if (obj is uint u32) {
return u32;
}
if (obj is ulong u64) {
return (long)u64;
}
if (obj is short i16) {
return i16;
}
if (obj is ushort u16) {
return u16;
}
}
return null;
}
public static DateTime? GetUnixEpochDateTime(this IBlueRow row, string colName) =>
GetUnixEpochDateTime(row[colName]);
public static DateTime? GetUnixEpochDateTime(this IBlueRow row, int colNo) =>
GetUnixEpochDateTime(row[colNo]);
public static DateTime? GetUnixEpochDateTime(object obj) {
var tsNanos = GetInt64(obj);
return tsNanos.HasValue ? DateTime.UnixEpoch.Add(new TimeSpan(tsNanos.Value / 100)) : null;
}
public static DateTime? GetDateTime(object obj) {
var res = _get_datetime_value(obj, DateTime.MinValue);
if (res == DateTime.MinValue) {
return null;
}
return res;
}
public static object GetEnum(Type type, object o) {
try {
if (o is null || o == DBNull.Value) {
return null;
} else if (o is string str) {
return Enum.TryParse(type, str, true, out var res) ? res : null;
} else if (o is int || o is uint || o is long || o is ulong || o is short || o is ushort) {
return Convert.ChangeType(o, type);
}
} catch {
}
return null;
}
public static T? GetEnum<T>(object o) where T : struct, Enum {
try {
if (o is null || o == DBNull.Value) {
return null;
} else if (o is string str) {
return Enum.TryParse(str, true, out T res) ? res : null;
} else if (o is int || o is short || o is ushort || o is uint || o is long || o is ulong)
return (T)o;
else if (o is T t) {
return t;
}
} catch {
}
return null;
}
public static T get_enum<T>(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct {
return BlueDataUtility.get_row_enum_value<T>(r, col_name, err_value);
}
@@ -0,0 +1,311 @@
#nullable enable
namespace Bts.DataStreamCore;
using BCG;
using mbts.utils;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
public class BlueRowStore<T1, T2, T3, T4, T5, T6> : IBlueRow, IReadOnlyDictionary<string, object?> {
readonly string? name;
IBlueTableSchema? _schema;
public BlueRowStore() { }
public BlueRowStore(
int id,
(string Name, T1 Value) item1,
(string Name, T2 Value) item2,
(string Name, T3 Value) item3,
(string Name, T4 Value) item4,
(string Name, T5 Value) item5,
(string Name, T6 Value) item6
) {
Id = id;
Item1 = item1;
Item2 = item2;
Item3 = item3;
Item4 = item4;
Item5 = item5;
Item6 = item6;
}
public int Id { get; set; }
public (string Name, T1 Value) Item1 { get; set; }
public (string Name, T2 Value) Item2 { get; set; }
public (string Name, T3 Value) Item3 { get; set; }
public (string Name, T4 Value) Item4 { get; set; }
public (string Name, T5 Value) Item5 { get; set; }
public (string Name, T6 Value) Item6 { get; set; }
public int Count => 5;
public IReadOnlyList<string> Keys => new KeyList(this);
public IReadOnlyList<object?> Values => new ValueList(this);
public IBlueTableSchema schema =>
_schema ??= new BlueTableSchema(name, new ColumnInfo[]{
new("id", 0, typeof(int)),
new(Item1.Name, 1, typeof(T1)),
new(Item2.Name, 2, typeof(T2)),
new(Item3.Name, 3, typeof(T3)),
new(Item4.Name, 4, typeof(T4)),
new(Item5.Name, 5, typeof(T5)),
new(Item6.Name, 6, typeof(T6)),
});
ICollection IDictionary.Keys => new KeyList(this);
ICollection IDictionary.Values => new ValueList(this);
IEnumerable<string> IReadOnlyDictionary<string, object?>.Keys => Keys;
IEnumerable<object?> IReadOnlyDictionary<string, object?>.Values => Values;
bool ICollection.IsSynchronized => false;
object ICollection.SyncRoot => this;
public object? this[string colName] {
get {
if (colName == "id") {
return Id;
}
if (colName == Item1.Name) {
return Item1.Value;
}
if (colName == Item2.Name) {
return Item2.Value;
}
if (colName == Item3.Name) {
return Item3.Value;
}
if (colName == Item4.Name) {
return Item4.Value;
}
if (colName == Item5.Name) {
return Item5.Value;
}
if (colName == Item6.Name) {
return Item6.Value;
}
throw new KeyNotFoundException();
}
set {
if (colName == "id") {
Id = ExceptionAssert.Cast(value).To<int>();
}
if (colName == Item1.Name) {
Item1 = (Item1.Name, ExceptionAssert.Cast(value).To<T1>());
}
if (colName == Item2.Name) {
Item2 = (Item2.Name, ExceptionAssert.Cast(value).To<T2>());
}
if (colName == Item3.Name) {
Item3 = (Item3.Name, ExceptionAssert.Cast(value).To<T3>());
}
if (colName == Item4.Name) {
Item4 = (Item4.Name, ExceptionAssert.Cast(value).To<T4>());
}
if (colName == Item5.Name) {
Item5 = (Item5.Name, ExceptionAssert.Cast(value).To<T5>());
}
if (colName == Item6.Name) {
Item6 = (Item6.Name, ExceptionAssert.Cast(value).To<T6>());
}
throw new KeyNotFoundException();
}
}
public object? this[int index] => index switch {
0 => Id,
1 => Item1.Value,
2 => Item2.Value,
3 => Item3.Value,
4 => Item4.Value,
5 => Item5.Value,
6 => Item6.Value,
_ => throw new IndexOutOfRangeException()
};
object? IDictionary.this[object key] {
get => this[ExceptionAssert.Cast(key).To<string>()];
set => this[ExceptionAssert.Cast(key).To<string>()] = value;
}
public void CopyTo(Array array, int index) {
array.SetValue(Id, index++);
array.SetValue(Item1.Value, index++);
array.SetValue(Item2.Value, index++);
array.SetValue(Item3.Value, index++);
array.SetValue(Item4.Value, index++);
array.SetValue(Item5.Value, index++);
array.SetValue(Item6.Value, index++);
}
public bool Contains(object key) =>
key is string keystr && ContainsKey(keystr);
public bool ContainsKey(string key) =>
Keys.Contains(key);
public void SetId(int id) =>
Id = id;
public void SetValue1(T1 value) =>
Item1 = (Item1.Name, value);
public void SetValue1(string colName, T1 value) =>
Item1 = (colName, value);
public void SetValue2(T2 value) =>
Item2 = (Item2.Name, value);
public void SetValue2(string colName, T2 value) =>
Item2 = (colName, value);
public void SetValue3(T3 value) =>
Item3 = (Item3.Name, value);
public void SetValue3(string colName, T3 value) =>
Item3 = (colName, value);
public void SetValue4(T4 value) =>
Item4 = (Item4.Name, value);
public void SetValue4(string colName, T4 value) =>
Item4 = (colName, value);
public void SetValue5(T5 value) =>
Item5 = (Item5.Name, value);
public void SetValue5(string colName, T5 value) =>
Item5 = (colName, value);
public void SetValue6(T6 value) =>
Item6 = (Item6.Name, value);
public void SetValue6(string colName, T6 value) =>
Item6 = (colName, value);
public bool TryGetValue(string key, [MaybeNullWhen(false)] out object? value) {
if (!Keys.TryFindIndex(key, out int index)) {
value = null;
return false;
}
value = this[index];
return true;
}
void IDictionary.Add(object key, object? value) =>
throw new NotSupportedException();
void IDictionary.Clear() =>
throw new NotSupportedException();
void IDictionary.Remove(object key) =>
throw new NotSupportedException();
IDictionaryEnumerator IDictionary.GetEnumerator() =>
throw new NotImplementedException();
IEnumerator IEnumerable.GetEnumerator() =>
throw new NotImplementedException();
public IEnumerator<KeyValuePair<string, object?>> GetEnumerator() {
throw new NotImplementedException();
}
class KeyList : ICollection, IReadOnlyList<string> {
readonly BlueRowStore<T1, T2, T3, T4, T5, T6> store;
public KeyList(BlueRowStore<T1, T2, T3, T4, T5, T6> store) =>
this.store = store;
public int Count => 5;
bool ICollection.IsSynchronized => false;
object ICollection.SyncRoot => this;
public string this[int index] =>
index switch {
0 => "id",
1 => store.Item1.Name,
2 => store.Item2.Name,
3 => store.Item3.Name,
4 => store.Item4.Name,
5 => store.Item5.Name,
6 => store.Item6.Name,
_ => throw new IndexOutOfRangeException()
};
public void CopyTo(Array array, int index) {
array.SetValue(store.Id, index++);
array.SetValue(store.Item1.Value, index++);
array.SetValue(store.Item2.Value, index++);
array.SetValue(store.Item3.Value, index++);
array.SetValue(store.Item4.Value, index++);
array.SetValue(store.Item5.Value, index++);
array.SetValue(store.Item6.Value, index++);
}
public IEnumerator<string> GetEnumerator() {
yield return "id";
yield return store.Item1.Name;
yield return store.Item2.Name;
yield return store.Item3.Name;
yield return store.Item4.Name;
yield return store.Item5.Name;
yield return store.Item6.Name;
}
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
}
class ValueList : ICollection, IReadOnlyList<object?> {
readonly BlueRowStore<T1, T2, T3, T4, T5, T6> store;
public ValueList(BlueRowStore<T1, T2, T3, T4, T5, T6> store) =>
this.store = store;
public object? this[int index] =>
index switch {
0 => store.Id,
1 => store.Item1.Value,
2 => store.Item2.Value,
3 => store.Item3.Value,
4 => store.Item4.Value,
5 => store.Item5.Value,
6 => store.Item6.Value,
_ => throw new IndexOutOfRangeException()
};
public int Count => 5;
public bool IsSynchronized => false;
public object SyncRoot => this;
public void CopyTo(Array array, int index) {
array.SetValue(store.Id, index++);
array.SetValue(store.Item1.Value, index++);
array.SetValue(store.Item2.Value, index++);
array.SetValue(store.Item3.Value, index++);
array.SetValue(store.Item4.Value, index++);
array.SetValue(store.Item5.Value, index++);
array.SetValue(store.Item6.Value, index++);
}
public IEnumerator<object?> GetEnumerator() {
yield return store.Id;
yield return store.Item1.Value;
yield return store.Item2.Value;
yield return store.Item3.Value;
yield return store.Item4.Value;
yield return store.Item5.Value;
yield return store.Item6.Value;
}
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
}
}
@@ -0,0 +1,12 @@
namespace Bts.DataStreamCore;
using BCG;
using mbts.utils;
using System;
public interface ITimeSeriesRow : IBlueRow {
int Id => ExceptionAssert.of_type<int>(this["id"]);
DateTime Timestamp { get; }
}
@@ -0,0 +1,27 @@
#nullable enable
namespace Bts.DataStreamCore;
using BCG;
using System.Collections.Generic;
using System.Linq;
public class CompositeDataStreamFactory : IDataStreamFactory {
readonly List<IDataStreamFactory> streamFactories;
public CompositeDataStreamFactory(IEnumerable<IDataStreamFactory> streamFactories) {
this.streamFactories = streamFactories.ToList();
foreach (var rsf in this.streamFactories.OfType<IRecursiveDataStreamFactory>()) {
rsf.OnCompositeFactory(this);
}
}
public IEnumerable<IDataStreamMetadata> GetSupportedStreams() =>
streamFactories.SelectMany(f => f.GetSupportedStreams());
public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) =>
streamFactories.Where(f => f.CanCreate(streamId))
.Select(f => f.Create(streamId, args))
.ExceptNull()
.FirstOrDefault();
}
@@ -0,0 +1,66 @@
#nullable enable
namespace Bts.DataStreamCore;
using BCG;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Linq;
public class DataStreamFactory : IDataStreamFactory {
readonly List<IDataStreamFactory> factories = new();
public static IDataStreamFactory Create(IEnumerable<IDataStreamFactory> streamFactories) {
var dsf = new DataStreamFactory();
dsf.factories.AddRange(streamFactories);
return dsf.factories.Count == 1 ? dsf.factories[0] : dsf;
}
public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) =>
factories.Where(sf => sf.GetSupportedStreams().Any(m => m.IsMatch(streamId)))
.Select(sf => sf.Create(streamId, args))
.ExceptNull()
.FirstOrDefault();
public IEnumerable<IDataStreamMetadata> GetSupportedStreams() =>
factories.SelectMany(sf => sf.GetSupportedStreams());
public void Add(IDataStreamFactory streamFactory) =>
factories.Add(streamFactory);
class EmptyDataStreamFactory : IDataStreamFactory {
public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) =>
null;
public IEnumerable<IDataStreamMetadata> GetSupportedStreams() =>
Enumerable.Empty<IDataStreamMetadata>();
}
}
public static class DataStreamFactoryExtensions {
public static void Add<TParams>(this DataStreamFactory streamFactory, IDataStreamMetadata metadata, Func<TParams, ViewStream?> singleStreamfactory, ILogger? logger = null) {
var sf = new SingleDataStreamFactory<TParams>(metadata, singleStreamfactory, logger);
streamFactory.Add(sf);
}
public static void Add<TParams>(this DataStreamFactory streamFactory, string name, Func<TParams, ViewStream?> singleStreamfactory, ILogger? logger = null) {
var sf = new SingleDataStreamFactory<TParams>(new DataStreamMetadata() { Name = name }, singleStreamfactory, logger);
streamFactory.Add(sf);
}
public static void Add(this DataStreamFactory streamFactory, string name, Func<IEnumerable<(string?, int?, object?)>, ViewStream?> singleStreamfactory) {
var sf = new SingleDataStreamFactory(new DataStreamMetadata() { Name = name }, singleStreamfactory);
streamFactory.Add(sf);
}
public static void Add<TParams>(this DataStreamFactory streamFactory, string[] names, Func<TParams, ViewStream?> singleStreamfactory, ILogger? logger = null) {
if (names.Length == 0) {
throw new ArgumentException("StreamFactory must have at least one name", nameof(names));
}
var meta = new DataStreamMetadata() { Name = names[0], AlternateNames = names.Skip(1).ToList() };
var sf = new SingleDataStreamFactory<TParams>(meta, singleStreamfactory, logger);
streamFactory.Add(sf);
}
}
@@ -0,0 +1,54 @@
#nullable enable
namespace Bts.DataStreamCore;
using BCG;
using bts;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
public interface IDataStreamMetadata {
public string Name { get; }
public IEnumerable<string> AlternateNames { get; }
}
public class DataStreamMetadata : IDataStreamMetadata {
public string? Name { get; set; }
public List<string>? AlternateNames { get; set; }
string IDataStreamMetadata.Name => Name ?? string.Empty;
IEnumerable<string> IDataStreamMetadata.AlternateNames =>
AlternateNames ?? new(0);
public static DataStreamMetadata FromType(Type type) {
var friendlyName = type.GetCustomAttribute<DataStreamUserRespresentation>()?.friendlyName;
var rv = new DataStreamMetadata() {
Name = friendlyName ?? type.Name,
AlternateNames = new(2) {
Log.notnull(type.FullName)
}
};
if (friendlyName is not null) {
rv.AlternateNames.Add(type.Name);
}
return rv;
}
}
public static class DataStreamMetadataExtensions {
public static bool IsMatch(this IDataStreamMetadata dsm, string name, bool checkAlternateNames = true, bool ignoreCase = true, bool ignoreUnderscores = true) {
var strComp = ignoreCase ? StringComparison.CurrentCultureIgnoreCase : StringComparison.CurrentCulture;
var nameWithoutUnderscores = new Lazy<string>(() => RemoveUnderscores(name));
return IsNameMatch(dsm.Name) || (checkAlternateNames && dsm.AlternateNames.Any(IsNameMatch));
static string RemoveUnderscores(string s) =>
s.Replace("_", "");
bool IsNameMatch(string otherName) =>
string.Equals(name, otherName, strComp)
|| (ignoreUnderscores && string.Equals(nameWithoutUnderscores.Value, RemoveUnderscores(otherName), strComp));
}
}
@@ -0,0 +1,25 @@
#nullable enable
namespace Bts.DataStreamCore;
using BCG;
using System.Collections.Generic;
using System.Linq;
public interface IDataStreamFactory {
IEnumerable<IDataStreamMetadata> GetSupportedStreams();
ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args);
bool CanCreate(string streamId) =>
GetSupportedStreams().Any(s => s.IsMatch(streamId));
}
/// <summary>
/// This interface provides a way for IDataStreamFactory's that create ViewStreams
/// that depend specifically on the composite IDataStreamFactory. This was designed
/// specifically for the NLStreamDataFactory.
/// </summary>
public interface IRecursiveDataStreamFactory : IDataStreamFactory {
void OnCompositeFactory(IDataStreamFactory compositeFactory);
}
@@ -0,0 +1,18 @@
#nullable enable
namespace Bts.DataStreamCore;
using BCG;
using System.Collections.Generic;
public interface ISingleStreamFactory : IDataStreamFactory {
IDataStreamMetadata Metadata { get; }
ViewStream? Create(IEnumerable<(string? Name, int? Index, object? Value)> args);
ViewStream? IDataStreamFactory.Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) =>
Metadata.IsMatch(streamId) ? Create(args) : null;
IEnumerable<IDataStreamMetadata> IDataStreamFactory.GetSupportedStreams() =>
new[] { Metadata };
}
@@ -0,0 +1,6 @@
#nullable enable
namespace Bts.DataStreamCore;
public interface IStreamArgumentParser {
bool TryParse(string? name, string? valueStr, out object? result);
}
@@ -8,33 +8,36 @@ namespace BCG {
using System.Collections;
using System.Reflection;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using bts.collections;
using System.Diagnostics.CodeAnalysis;
using Spark.Core.Collections;
using Bts.DataStreamCore;
using Microsoft.Extensions.Options;
public class PrimitiveManager {
static MarketDataStreamFactory mdStreamFactory;
private IDictionary myTypeLookupTable = new Hashtable();
public class PrimitiveManager : IDataStreamFactory {
private static PrimitiveManager mySelf;
private ArrayList myAssemblyNames = new ArrayList();
readonly List<string> myAssemblyNames = new();
readonly Dictionary<string, Type> myTypeLookupTable = new();
private PrimitiveManager() {
myLoadAllTypes();
}
public PrimitiveManager(IOptions<PrimitiveManagerConfig> config) : this(config.Value) { }
private void myLoadAllTypes() {
public PrimitiveManager(PrimitiveManagerConfig config) {
AppDomain currentDomain = AppDomain.CurrentDomain;
currentDomain.AssemblyResolve += new ResolveEventHandler(myAssemblyResolver);
loadAssembly(this.GetType().Assembly.FullName);
var assemblies = new string[] { "DataStream.dll", "SharedStreams.dll" };
var config_assemblies = Configuration.get<DataStreamConfig>().Assemblies;
bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes config_assemblies: {config_assemblies}");
foreach (string assemblyName in assemblies/*config_assemblies*/) {
foreach (string assemblyName in config.Assemblies.empty_if_null()) {
bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes assemblyName: {assemblyName}");
string resolvedAssemblyPath = assemblyName;
if (!System.IO.Path.IsPathRooted(assemblyName)) {
resolvedAssemblyPath = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assemblyName));
}
loadAssembly(resolvedAssemblyPath);
loadAssembly(resolvedAssemblyPath, IncludeType);
}
bool IncludeType(Type t) =>
!config.ExcludeTypes.empty_if_null().Any(tname => tname == t.Name);
}
private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) {
@@ -48,12 +51,7 @@ namespace BCG {
/// Loads all the Stream types associated with the assembly
/// </summary>
/// <param name="assemblyName"></param>
public void loadAssembly(string assemblyName) {
Func<Type, bool> is_blue_orb = (t) => {
//if (!assemblyName.Contains("SharedStreams.dll"))
// return true;
return t.Namespace == "bts.BlueOrb";// || t.Namespace == "BCG";
};
public void loadAssembly(string assemblyName, Func<Type, bool> typeFilter = null) {
bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
if (myAssemblyNames.Contains(assemblyName))
return;
@@ -62,8 +60,12 @@ namespace BCG {
GetType().Assembly :
Assembly.LoadFrom(assemblyName);
foreach (Type type in loadedAssembly.GetTypes())
if (!type.IsInterface && is_blue_orb(type)
IEnumerable<Type> types = loadedAssembly.GetTypes();
if (typeFilter is not null) {
types = types.Where(typeFilter);
}
foreach (Type type in types)
if (!type.IsInterface
&& !type.IsAbstract
&& type.IsSubclassOf(typeof(ViewStream))
&& type.IsPublic) {
@@ -79,7 +81,7 @@ namespace BCG {
break;
}
if (isVisible) {
// Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
Log.assert_d(!myTypeLookupTable.ContainsKey(userRep));
myTypeLookupTable[userRep] = type;
}
}
@@ -95,26 +97,28 @@ namespace BCG {
return retVal;
}
public IEnumerable<Type> GetSupportedTypes() =>
myTypeLookupTable.Values.Cast<Type>();
public IEnumerable<IDataStreamMetadata> GetSupportedStreams() =>
GenericDictionaryWrapper.Create<string, Type>(myTypeLookupTable).Select(kv => new DataStreamMetadata() {
Name = kv.Key,
AlternateNames = new() { kv.Value.Name, kv.Value.FullName }
});
public static PrimitiveManager instance {
get { return mySelf = mySelf == null ? new PrimitiveManager() : mySelf; }
}
public Type ResolveType(string streamId) =>
TryResolveType(streamId, out var t) ? t
: throw new ArgumentException($"Failed to resolve ViewStream type '{streamId}'", nameof(streamId));
public static void SetMarketDataClient(object value) {
Log.isnull(mdStreamFactory);
mdStreamFactory = new(value);
}
/// <summary>
/// Returns the Type name corresponding to the primitive name
/// </summary>
/// <param name="friendlyName"></param>
/// <returns></returns>
public string find(string friendlyName) {
if (myTypeLookupTable[friendlyName] == null)
throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
return (myTypeLookupTable[friendlyName] as Type).Name;
public bool TryResolveType(string streamId, [NotNullWhen(true)] out Type result) {
result = GetSupportedStreams()
.Where(s => s.IsMatch(streamId))
.Select(s => s.Name)
.Prepend(streamId)
.Select(myTypeLookupTable.GetValueOrDefault)
.ExceptNull()
.FirstOrDefault();
return result is not null;
}
/// <summary>
@@ -130,19 +134,110 @@ namespace BCG {
constructorArgs = new object[args.Count];
args.CopyTo(constructorArgs, 0);
}
var type = myTypeLookupTable[friendlyName] as Type
?? throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
if (type.HasCustomAttribute<LiveMarketDataStreamAttribute>()) {
if (mdStreamFactory is null) {
Log.Gen.error("Live md not available for streams");
return null;
}
return mdStreamFactory.Create(type, constructorArgs);
if (myTypeLookupTable[friendlyName] == null)
throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
try {
return
constructorArgs == null ?
Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type) as ViewStream :
Activator.CreateInstance(myTypeLookupTable[friendlyName] as Type, constructorArgs) as ViewStream;
} catch (Exception ex) {
Log.Gen.warn($"PrimitiveManager failed to construct ViewStream type '{friendlyName}': {ex}");
}
return null;
}
public ViewStream Create(string streamId, IEnumerable<(string Name, int? Index, object Value)> args) =>
TryResolveType(streamId, out var type) ? Create(type, args) : null;
/// <summary>
/// Constructs and returns a ViewStream of type corresponding to the friendly name
/// </summary>
/// <param name="friendlyName">friendly primitive name </param>
/// <param name="args">args can be null or an empty ICollection if the paramterless constructor
/// is the desired one</param>
/// <returns></returns>
public ViewStream Create(Type type, IEnumerable<(string Name, int? Index, object Value)> args) {
var ctrs = type.GetConstructors(BindingFlags.Instance | BindingFlags.Public);
var argList = args.ToList();
const int MAX_ALLOWED_ARGS = 64;
if (argList.Count > MAX_ALLOWED_ARGS) {
throw new ArgumentException(
$"ViewStream arg count ({argList.Count}) exceeds max allowed ({MAX_ALLOWED_ARGS})", nameof(args));
}
if (argList.Any(a => !(a.Name?.Length > 0) && !a.Index.HasValue)) {
throw new ArgumentException(
"One or more ViewStream args lack both Name and Index (unable to assign a constructor param)", nameof(args));
}
var indexedArgs = argList.Where(a => a.Index.HasValue).ToList();
var namedArgs = argList.Where(a => a.Name?.Length > 0).ToList();
if (!namedArgs.Select(a => a.Name).IsDistinct()) {
throw new ArgumentException("ViewStream arg names are not unique");
}
if (!indexedArgs.Select(a => a.Index.Value).IsDistinct()) {
throw new ArgumentException("ViewStream arg indices are not unique");
}
// Constructor params with default values may be omitted from the input 'args'. Therefore,
// 'minArgCount' >= 'argList.Count'.
var minArgCount = Math.Max(indexedArgs.Select(a => a.Index.Value).MaxOrDefault(), argList.Count);
foreach (var ctr in ctrs) {
if (TryAssignConstructorArgs(ctr, minArgCount, argList, indexedArgs, namedArgs, out var ctrArgs)) {
return ctr.Invoke(ctrArgs) as ViewStream;
}
}
return null;
static bool TryAssignConstructorArgs(
ConstructorInfo ctr,
int minArgCount,
List<(string Name, int? Index, object Value)> argList,
List<(string Name, int? Index, object Value)> indexedArgs,
List<(string Name, int? Index, object Value)> namedArgs,
[NotNullWhen(true)] out object[] ctrArgs
) {
ctrArgs = null;
var ctrParams = ctr.GetParameters();
if (ctrParams.Length < minArgCount) {
return false;
}
var ctrArgList = new List<object>();
int remInputArgs = argList.Count;
for (int i = 0; i < ctrParams.Length; i++) {
var ctrParam = ctrParams[i];
var paramType = ctrParams[i].ParameterType;
if (indexedArgs.TryGetFirst(a => CanAssignInputArg(ctrParam, a), out var a)) {
remInputArgs--;
ctrArgList.Add(a.Value);
} else if (ctrParam.HasDefaultValue) {
ctrArgList.Add(ctrParam.DefaultValue);
} else {
return false;
}
}
if (remInputArgs > 0) {
return false;
}
Log.assert(ctrArgList.Count == ctrParams.Length);
ctrArgs = ctrArgList.ToArray();
return true;
}
static bool CanAssignInputArg(ParameterInfo pi, (string Name, int? Index, object Value) arg) {
if (!string.IsNullOrEmpty(arg.Name) && arg.Name != pi.Name) {
return false;
}
if (arg.Index.HasValue && arg.Index.Value != pi.Position) {
return false;
}
if (arg.Value is null) {
if (!pi.ParameterType.IsValueType && !pi.ParameterType.IsNullable()) {
return false;
}
} else if (!pi.ParameterType.IsAssignableFrom(arg.Value.GetType())) {
return false;
}
return true;
}
return
constructorArgs == null ?
Activator.CreateInstance(type) as ViewStream :
Activator.CreateInstance(type, constructorArgs) as ViewStream;
}
/// <summary>
@@ -0,0 +1,9 @@
#nullable enable
namespace Bts.DataStreamCore;
using System.Collections.Generic;
public class PrimitiveManagerConfig {
public List<string>? Assemblies { get; set; }
public List<string>? ExcludeTypes { get; set; }
}
@@ -0,0 +1,59 @@
#nullable enable
namespace Bts.DataStreamCore;
using BCG;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
class SingleDataStreamFactory : ISingleStreamFactory {
readonly Func<IEnumerable<(string?, int?, object?)>, ViewStream?> factory;
public SingleDataStreamFactory(IDataStreamMetadata metadata, Func<IEnumerable<(string?, int?, object?)>, ViewStream?> factory) {
Metadata = metadata;
this.factory = factory;
}
public IDataStreamMetadata Metadata { get; }
public ViewStream? Create(IEnumerable<(string? Name, int? Index, object? Value)> args) =>
factory(args);
public IEnumerable<IDataStreamMetadata> GetSupportedStreams() =>
new[] { Metadata };
}
class SingleDataStreamFactory<TParams> : ISingleStreamFactory {
readonly Func<TParams, ViewStream?> factory;
readonly ILogger? logger;
readonly StreamFactoryArgStore.TryGetParamsDelegate<TParams> tryGetParams;
public SingleDataStreamFactory(IDataStreamMetadata metadata, Func<TParams, ViewStream?> factory)
: this(metadata, factory, null) { }
public SingleDataStreamFactory(IDataStreamMetadata metadata, Func<TParams, ViewStream?> factory, ILogger? logger, StreamFactoryArgStore.TryGetParamsDelegate<TParams>? tryGetParams = null) {
Metadata = metadata;
this.factory = factory;
this.logger = logger;
this.tryGetParams = tryGetParams ?? new(StreamFactoryArgStore.TryGetParams);
}
public IDataStreamMetadata Metadata { get; }
public ViewStream? Create(IEnumerable<(string? Name, int? Index, object? Value)> args) {
if (!tryGetParams(args, out TParams? streamParams, out var err)) {
logger?.LogInvalidParams(Metadata.Name, err);
return null;
}
return Create(streamParams);
}
public ViewStream? Create(TParams streamParams) {
return factory(streamParams);
}
public IEnumerable<IDataStreamMetadata> GetSupportedStreams() =>
new[] { Metadata };
}
@@ -0,0 +1,19 @@
#nullable enable
namespace Bts.DataStreamCore;
using System;
[AttributeUsage(AttributeTargets.Property)]
public class StreamArgumentAttribute : Attribute {
public StreamArgumentAttribute(params string[] names) {
Names = names;
RegexNames = Array.Empty<string>();
}
public string[] Names { get; }
public string[] RegexNames { get; set; }
public int Position { get; set; } = -1;
public bool IsRequired { get; set; }
public bool AllowCommaSeparatedAliases { get; set; } = true;
public Type? ParserType { get; set; }
}
@@ -0,0 +1,329 @@
#nullable enable
namespace Bts.DataStreamCore;
using BCG;
using mbts.utils;
using Microsoft.Extensions.Logging;
using Spark.Core;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
public class StreamFactoryArgStore {
readonly ILogger? logger;
readonly string streamType;
ImmutableList<(string? Name, int? Index, object? Value)> args;
public StreamFactoryArgStore(IEnumerable<(string? Name, int? Index, object? Value)> args, string? streamType = null, ILogger? logger = null) {
this.args = args.ToImmutableList();
this.streamType = streamType ?? "anonStream";
this.logger = logger;
}
public delegate bool TryGetParamsDelegate<T>(IEnumerable<(string? Name, int? Index, object? Value)> args, [NotNullWhen(true)]out T? result, [NotNullWhen(false)] out string? errorMessage);
public delegate bool TryConvert<T>(object? obj, [NotNullWhen(true)]out T? result);
public static bool TryGetParams<T>(
IEnumerable<(string? Name, int? Index, object? Value)> args,
[NotNullWhen(true)] out T? result,
[NotNullWhen(false)] out string? errorMessage
) =>
new StreamFactoryArgStore(args).TryGetParams(out result, out errorMessage);
public bool TryGetParams<T>([MaybeNull][NotNullWhen(true)] out T result, [NotNullWhen(false)] out string? errorMessage) {
var props = typeof(T).GetProperties();
var res = Activator.CreateInstance<T>();
if (res is null) {
errorMessage = $"Failed to instantiate {typeof(T)}";
result = default;
return false;
}
foreach (var p in props) {
var att = p.GetCustomAttribute<StreamArgumentAttribute>();
IEnumerable<string> names = att?.Names ?? new[] { p.Name };
if (att?.AllowCommaSeparatedAliases ?? false) {
names = names.SelectMany(n => n.Split(','));
}
IEnumerable<string> regexNames = att?.RegexNames ?? Enumerable.Empty<string>();
if (TryGet(names, out var arg) || TryGetRegex(regexNames, out arg)) {
SetValue(arg);
} else if (att is not null && att.Position >= 0 && args.Skip(att.Position).TryGetFirst(out arg)) {
SetValue(arg);
} else if (att?.IsRequired ?? false) {
errorMessage = $"Missing required param '{p.Name}'";
result = default;
return false;
}
void SetValue((string? Name, int? Index, object? Value) arg) {
var (name, index, val) = arg;
if (val is string valStr && att?.ParserType is Type parserType) {
var parser = ExceptionAssert.of_type<IStreamArgumentParser>(Activator.CreateInstance(parserType));
if (!parser.TryParse(name, valStr, out val)) {
throw new InvalidOperationException($"Failed to parse arg with custom parser of type {parserType}");
}
}
var pType = p.PropertyType;
bool isNullable = pType.IsNullable();
if (isNullable) {
pType = pType.GetGenericArguments().Single();
}
var vType = val?.GetType() ?? typeof(object);
if (pType == typeof(DateTime)) {
SetValueType(BlueDataUtility.GetDateTime(val));
} else if (pType == typeof(TimeSpan)) {
SetValueType(BlueDataUtility.GetTimeSpan(val));
} else if (pType == typeof(string)) {
p.SetValue(res, val as string ?? val?.ToString());
} else if (pType.IsEnum) {
p.SetValue(res, BlueDataUtility.GetEnum(pType, val));
} else if (vType.IsAssignableTo(pType) || (val is null && (!pType.IsValueType || isNullable))) {
p.SetValue(res, val);
} else {
p.SetValue(res, Convert.ChangeType(val, pType));
}
void SetValueType<TProp>(TProp? pVal) where TProp : struct {
if (isNullable) {
p.SetValue(res, pVal);
} else {
p.SetValue(res, pVal.assert_value());
}
}
}
}
errorMessage = null;
result = res;
return true;
}
public bool GetInt32(string name, [NotNullWhen(true)] out int? result) =>
GetInt32(out result, true, name);
public bool GetInt32([NotNullWhen(true)] out int? result, bool required, params string[] names) {
result = GetInt32(names);
if (required && !result.HasValue) {
logger.LogMissingParam(streamType, names.Length switch {
0 => "Int32",
_ => names[0],
});
}
return result.HasValue;
}
public int? GetInt32(params string[] names) =>
GetInt32(true, names);
public int? GetInt32(bool required, params string[] names) {
if (names.Length == 0) {
return null;
}
var obj = Get(names);
if (obj is int i) {
return i;
}
var s = obj as string ?? obj?.ToString();
if (int.TryParse(s, out i)) {
return i;
}
if (required && logger is not null) {
logger.LogMissingParam(streamType, names[0]);
}
return null;
}
public bool GetTimeSpan(string name, out TimeSpan? result) {
result = GetTimeSpan(true, new[] { name });
return result.HasValue;
}
public TimeSpan? GetTimeSpan(string name) =>
GetTimeSpan(true, new[] { name });
public TimeSpan? GetTimeSpan(bool required, string[] preferredNames) =>
preferredNames
.Select(n => BlueDataUtility.GetTimeSpan(Get(n)))
.Where(ts => ts.HasValue)
.FirstOrDefault();
public IEnumerable<string> GetStrings() =>
args.Select(a => a.Value).OfType<string>();
// First, parses the args associated with preferredNames and returns the first successful
// parse result. If none were successful, tries to parse every arg.
public T? GetEnum<T>(params string[] preferredNames) where T : struct {
var candidateObjs = preferredNames.Select(n => Get(n)).Concat(args.Select(a => a.Value));
foreach (var obj in candidateObjs) {
if (obj is T res) {
return res;
}
if (obj is string s && Enum.TryParse(s, true, out res)) {
return res;
}
}
return null;
}
public IEnumerable<DateTime> GetDateTimes() =>
args.Select(a => convertToDateTime(a.Value)).SelectWhereValues();
public bool GetDateTime(string name, [NotNullWhen(true)] out DateTime? result) {
result = GetDateTime(true, name);
return result.HasValue;
}
public bool GetDateTime([NotNullWhen(true)] out DateTime? result, params string[] names) {
result = GetDateTime(names);
return result.HasValue;
}
public bool GetDateTime([NotNullWhen(true)] out DateTime? result, out int index, bool removeArg) {
int i = 0;
foreach (var arg in args) {
var dt = convertToDateTime(arg.Value);
if (dt.HasValue) {
result = dt.Value;
index = i;
if (removeArg) {
args = args.RemoveAt(i);
}
return true;
}
}
result = null;
index = -1;
return false;
}
public DateTime? GetDateTime(params string[] names) =>
GetDateTime(true, names);
public DateTime? GetDateTime(bool required, params string[] names) =>
names.Select(n => Get(n)).Select(convertToDateTime).Where(dt => dt.HasValue).FirstOrDefault();
public object? Get(params string[] names) =>
Get(true, names);
public object? Get(out int index, params string[] names) =>
Get(true, out index, names);
public object? Get(bool required, params string[] names) =>
Get(required, out _, names);
public object? Get(bool required, out int index, params string[] names) {
if (names.Length == 0) {
index = -1;
return null;
}
if (!TryGet(names, out var result, out index) && required && logger is not null) {
logger.LogMissingParam(streamType, names[0]);
}
return result.Value;
}
public bool TryGetRegex(string name, out (string? Name, int? Index, object? Value) result) =>
TryGet(new[] { name }, out result, checkDefaultMatch: false, checkRegexMatch: true);
public bool TryGetRegex(IEnumerable<string> names, out (string? Name, int? Index, object? Value) result) =>
TryGet(names, out result, checkDefaultMatch: false, checkRegexMatch: true);
public bool TryGet(IEnumerable<string> names, out (string? Name, int? Index, object? Value) result, bool checkDefaultMatch = true, bool checkRegexMatch = false) =>
TryGet(names, out result, out _, checkDefaultMatch, checkRegexMatch);
public bool TryGet(IEnumerable<string> names, out (string? Name, int? Index, object? Value) result, out int index, bool checkDefaultMatch = true, bool checkRegexMatch = false) {
if (checkDefaultMatch) {
foreach (var name in names.SelectMany(Variants)) {
if (args.TryGetFirst(a => a.Name == name, out var a, out index)) {
result = a;
return true;
}
}
}
if (checkRegexMatch) {
foreach (var name in names) {
if (args.TryGetFirst(a => Regex.IsMatch(a.Name ?? string.Empty, name, RegexOptions.IgnoreCase), out var a, out index)) {
result = a;
return true;
}
}
}
index = -1;
result = default;
return false;
IEnumerable<string> Variants(string s) {
if (string.IsNullOrEmpty(s)) {
yield break;
}
yield return s;
// Convert to snake_case
var bd = new StringBuilder(char.ToLower(s[0]));
for (int i = 1; i < s.Length; i++) {
if (i > 1 && char.IsUpper(s[i]) && char.IsLower(s[i - 1])) {
bd.Append('_');
}
bd.Append(char.ToLower(s[i]));
}
yield return bd.ToString();
// Convert to camelCase
bd.Clear();
for (int i = 0; i < s.Length; i++) {
int numUnderscores = 0;
while (s[i + numUnderscores] == '_' & (i + numUnderscores < s.Length)) {
numUnderscores++;
}
if (numUnderscores > 0) {
i += numUnderscores;
if (i >= s.Length) {
break;
}
bd.Append(char.ToUpper(s[i]));
} else {
bd.Append(s[i]);
}
}
yield return bd.ToString();
}
}
static DateTime? convertToDateTime(object? obj) =>
tryConvert(obj, out DateTime dt) ? dt : null;
static bool tryConvert(object? obj, out DateTime res) {
if (obj is DateTime dt) {
res = dt;
return true;
}
if (obj is string s) {
// if (OffsetDateTimePattern)
if (DateTimeOffset.TryParseExact(s, "yyyy-MM-ddTHH:mm:ssK", null, System.Globalization.DateTimeStyles.None, out var dto)) {
res = dto.ToUniversalTime().DateTime;
return true;
}
if (DateTimeOffset.TryParseExact(s, "yyyy-MM-ddTHH:mm:ss.fffffffffK", null, System.Globalization.DateTimeStyles.None, out dto)) {
res = dto.ToUniversalTime().DateTime;
return true;
}
if (DateTime.TryParse(s, null, System.Globalization.DateTimeStyles.RoundtripKind, out res)) {
return true;
}
if (DateTimeOffset.TryParse(s, out var resOff)) {
res = resOff.ToUniversalTime().DateTime;
return true;
}
return DateTime.TryParse(s, out res);
}
res = default;
return false;
}
}
@@ -0,0 +1,11 @@
namespace Bts.DataStreamCore;
using Microsoft.Extensions.Logging;
public static partial class StreamFactoryLogs {
[LoggerMessage(EventId = 1, Level = LogLevel.Warning, Message = "Unable to create '{StreamType}' stream. Missing required parameter '{Parameter}'.")]
public static partial void LogMissingParam(this ILogger logger, string streamType, string parameter);
[LoggerMessage(EventId = 2, Level = LogLevel.Warning, Message = "Unable to create '{StreamType}' stream. {error}.")]
public static partial void LogInvalidParams(this ILogger logger, string streamType, string error);
}
@@ -0,0 +1,58 @@
#nullable enable
namespace Bts.DataStreamCore;
public static class WellKnownStreamArgumentAliases {
/// <summary>
/// Aliases for a Venue argument.
/// </summary>
public const string Venue = "venue,exchange,exch";
/// <summary>
/// Aliases for a Type argument.
/// </summary>
public const string Asset = "symbol,sym,product,asset";
/// <summary>
/// Aliases for a Type argument.
/// </summary>
public const string Type = "type,typ";
/// <summary>
/// Aliases for a SecurityType argument. Some are potentially ambiguous, like 'type' and 'typ'.
/// If this conflicts with other arguments, consider using <see cref="QualifiedSecurityType"/>.
/// </summary>
public const string SecurityType = "secType,secTyp,securityType,securityTyp,type,typ";
/// <summary>
/// Aliases for a SecurityName argument. Some are potentially ambiguous, like 'name'. If this
/// conflicts with other arguments, consider using <see cref="QualifiedSecurityType"/>.
/// </summary>
public const string SecurityName = "name,sec,secName,security";
public const string QualifiedSecurityType = "secType,secTyp,securityType,securityTyp";
public const string QualifiedSecurityName = "sec,secName,security";
public const string VenueRegex = "^(venue|exchange|exch)$";
public const string AssetRegex = "^(symbol|sym|asset|product)$";
public const string TypeRegex = "^(type|typ)$";
public const string SecurityTypeRegex = "^(secType|secTyp|type|typ)$";
public const string SecurityNameRegex = "^(sec|security|name|secName|securityName)$";
public const string ShowVenueRegex = "^(show|include)(Exch|Venue)$";
public const string ShowAssetRegex = "^(show|include)(Symbol|Sym|Asset|Product)$";
public const string ShowSecurityTypeRegex = "^(show|include)(Sec|Security)?(Type|Typ)$";
public const string ShowSecurityNameRegex = "^(show|include)(Sec|Security|Name|SecName|SecurityName)$";
public const string ShowExpirationRegex = "^(show|include)(Exp(iration|Date)?)$";
public const string ShowCashPerPointRegex = "^(show|include)(CashPerPoint|Cpp)$";
public const string ShowLegVenueRegex = "^(show|include)Leg(Exch|Venue)$";
public const string ShowLegSecurityTypeRegex = "^(show|include)Leg(Sec|Security)?(Type|Typ)$";
public const string ShowLegAssetRegex = "^(show|include)Leg(Symbol|Sym|Asset|Product)$";
public const string ShowLegExpirationRegex = "^(show|include)Leg(Exp(iration|Date)?)$";
public const string ShowLegNameRegex = "^(show|include)Leg(Name|Desc)$";
public const string ShowLegCashPerPointRegex = "^(show|include)Leg(CashPerPoint|Cpp)$";
public const string ReplaySpeedRegex = $"^(replay|replaySpeed|speed|rate|velocity)$";
public const string BeginDateRegex = "(begin|start)(date(Time)?|time)?";
public const string EndDateRegex = "(end|stop)(date(Time)?|time)?";
}
@@ -3,6 +3,7 @@ namespace Bts.DataStreamCore.Services;
using BCG;
using BCG.Utilities.General;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
public static class DataStreamCoreServices {
@@ -14,4 +15,11 @@ public static class DataStreamCoreServices {
ViewStreamDatabase.SetGlobalInstance(new ViewStreamDatabase(sp.GetRequiredService<ISubscriberCore>()));
return ViewStreamDatabase.instance;
});
public static IServiceCollection AddPrimitiveStreams(this IServiceCollection services, IConfiguration configurationSection) =>
services.Configure<PrimitiveManagerConfig>(configurationSection)
.AddSingleton<IDataStreamFactory, PrimitiveManager>();
public static IServiceCollection AddCompositeDataStreamFactory(this IServiceCollection services) =>
services.AddSingleton<CompositeDataStreamFactory>();
}
@@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions;
using BCG;
using Bts.DataStreamCore;
class Assignment : StatementExpression {
readonly StatementExpression expression;
readonly string id;
@@ -19,8 +21,8 @@ class Assignment : StatementExpression {
return result;
}
public override ViewStream? myGetView(IDictionary<string, StatementExpression> env) {
var result = expression.getView(env);
public override ViewStream? myGetView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) {
var result = expression.getView(env, streamFactory);
env[id] = expression;
return result;
}
@@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions;
using BCG;
using Bts.DataStreamCore;
using System.Text;
public class DataStreamExpression {
@@ -31,10 +33,10 @@ public class DataStreamExpression {
return sb.ToString();
}
public ViewStream? getView(IDictionary<string, StatementExpression> env) {
public ViewStream? getView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) {
for (int i = 0; i < Statements.Count - 1; i++) {
Statements[i].getView(env);
Statements[i].getView(env, streamFactory);
}
return Statements[^1].getView(env);
return Statements[^1].getView(env, streamFactory);
}
}
@@ -2,7 +2,8 @@ namespace Bts.DataStreamExpressions;
using BCG;
using System.Collections;
using Bts.DataStreamCore;
using System.Collections.Immutable;
public class PrimitiveExpression : StatementExpression {
@@ -20,7 +21,7 @@ public class PrimitiveExpression : StatementExpression {
}
public override string print(int startCol, IList<string> activeBindings) {
string prelude = "new " + PrimitiveManager.instance.find(myName) + " (";
string prelude = "new " + myName + " (";
return prelude + myPrintArgs(startCol + prelude.Length, activeBindings) + ")";
}
@@ -31,21 +32,19 @@ public class PrimitiveExpression : StatementExpression {
return prelude + myPrintViewArgs(startCol + prelude.Length) + ")";
}
public override ViewStream myGetView(IDictionary<string, StatementExpression> env) {
ArrayList resolvedArgs = new ArrayList();
public override ViewStream? myGetView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) {
var resolvedArgs = myArgs.Select((arg, i) => ((string?)arg.Name, string.IsNullOrEmpty(arg.Name) ? (int?)i : null, ResolveArg(arg)));
return streamFactory.Create(myName, resolvedArgs);
foreach (var arg in myArgs) {
object? ResolveArg(Argument arg) {
if (arg.StatementExp is not null) {
resolvedArgs.Add(arg.StatementExp.myGetView(env));
continue;
return arg.StatementExp.myGetView(env, streamFactory);
}
if (arg.QuotedView is not null) {
resolvedArgs.Add(arg.QuotedView.Value);
continue;
return arg.QuotedView.Value;
}
resolvedArgs.Add(arg.Obj);
return arg.Obj;
}
return PrimitiveManager.instance.build(myName, resolvedArgs);
}
string myPrintSingleArg(int atColumn, object? o, IList<string> activeBindings) {
@@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions;
using BCG;
using Bts.DataStreamCore;
public abstract class StatementExpression {
ViewStream? view;
@@ -13,10 +15,10 @@ public abstract class StatementExpression {
public virtual string? printView(int startCol) =>
null;
public ViewStream? getView(IDictionary<string, StatementExpression> env) =>
view ??= myGetView(env);
public ViewStream? getView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) =>
view ??= myGetView(env, streamFactory);
public virtual ViewStream? myGetView(IDictionary<string, StatementExpression> env) =>
public virtual ViewStream? myGetView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) =>
null;
public static string makeCSString(string s) {
@@ -2,6 +2,8 @@ namespace Bts.DataStreamExpressions;
using BCG;
using Bts.DataStreamCore;
public class ViewNameExpression : StatementExpression {
readonly string id;
@@ -14,6 +16,6 @@ public class ViewNameExpression : StatementExpression {
public override string printView(int startCol) =>
id;
public override ViewStream? myGetView(IDictionary<string, StatementExpression> env) =>
env.TryGetValue(id, out var exp) ? exp.getView(env) : new TableStream(id);
public override ViewStream? myGetView(IDictionary<string, StatementExpression> env, IDataStreamFactory streamFactory) =>
env.TryGetValue(id, out var exp) ? exp.getView(env, streamFactory) : new TableStream(id);
}
@@ -4,6 +4,7 @@ using BCG;
using bts;
using Bts.DataStream;
using Bts.DataStreamCore;
using Bts.DataStreamExpressions;
using Bts.PubSub;
@@ -28,13 +29,28 @@ public sealed class DataStreamPublisherService : IHostedService, IDisposable {
readonly DataStreamPublisherConfig config;
readonly IPubTableBroker pubTableBroker;
readonly IWorkScheduler workScheduler;
readonly IDataStreamFactory streamFactory;
List<IDisposable>? resources = new();
public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, IOptions<DataStreamPublisherConfig> config)
: this(pubTableBroker, workScheduler, config.Value) { }
public DataStreamPublisherService(
IPubTableBroker pubTableBroker,
IEnumerable<IDataStreamFactory> streamFactories,
IWorkScheduler workScheduler,
IOptions<DataStreamPublisherConfig> config
)
: this(pubTableBroker, streamFactories, workScheduler, config.Value) { }
public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, DataStreamPublisherConfig config) {
public DataStreamPublisherService(
IPubTableBroker pubTableBroker,
IEnumerable<IDataStreamFactory> streamFactories,
IWorkScheduler workScheduler,
DataStreamPublisherConfig config
) {
this.pubTableBroker = pubTableBroker;
var streamFactoryList = streamFactories.ToList();
streamFactory = streamFactoryList.Count == 1
? streamFactoryList[0]
: new CompositeDataStreamFactory(streamFactoryList);
this.workScheduler = workScheduler;
this.config = config;
}
@@ -48,7 +64,7 @@ public sealed class DataStreamPublisherService : IHostedService, IDisposable {
Log.Gen.warn("Skipping PublishedStream because it doesn't have a name (must define 'Name' in config json).");
continue;
}
var vs = CreateViewStream(publishedStream);
var vs = CreateViewStream(streamFactory, publishedStream);
if (vs is null) {
continue;
}
@@ -72,7 +88,7 @@ public sealed class DataStreamPublisherService : IHostedService, IDisposable {
resources?.Clear();
}
static ViewStream? CreateViewStream(PublishedStream ps) {
static ViewStream? CreateViewStream(IDataStreamFactory streamFactory, PublishedStream ps) {
Log.Gen.info(
$"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
if (string.IsNullOrEmpty(ps.Name)) {
@@ -80,7 +96,7 @@ public sealed class DataStreamPublisherService : IHostedService, IDisposable {
$"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
return null;
}
var vs = buildStream(ps.Expression);
var vs = buildStream(streamFactory, ps.Expression);
if (vs is null) {
Log.Gen.warn(
$"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
@@ -89,14 +105,14 @@ public sealed class DataStreamPublisherService : IHostedService, IDisposable {
return vs;
}
static ViewStream? buildStream(string? streamExpression) {
static ViewStream? buildStream(IDataStreamFactory streamFactory, string? streamExpression) {
if (streamExpression is null) {
return null;
}
try {
Log.Gen.info($"Building ViewStream: {streamExpression}");
var exp = DataStreamExpressionsInterpreter.Evaluate(streamExpression);
return exp.getView(new Dictionary<string, StatementExpression>(0));
return exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
} catch (Exception ex) {
Log.assert(false, $"Could not build stream. expression: {streamExpression}; ex: {ex}");
return null;
@@ -19,6 +19,7 @@ public static class DataStreamPublisherServices {
.AddPubSubClient(configurationSection.GetSection("PubSub"))
.AddPubTableBroker()
.AddRemoteQueryManager()
.AddPrimitiveStreams(configurationSection.GetSection("PrimitiveStream"))
.Configure<DataStreamPublisherConfig>(configurationSection.GetSection("DataStreamPublisher"))
.AddHostedService<DataStreamPublisherService>();
}
@@ -22,8 +22,7 @@ using Bts.DataStreamExpressions;
// using BCG.Evaluator.DataStreamExpression;
// using BCG.Utilities.Win32;
using System.Threading.Tasks;
using Bts.DataStreamCore;
namespace BCG
{
@@ -49,9 +48,16 @@ namespace BCG
private ViewStream? myAllTagsStream;
private string myKQ;
private TextWriter? myConsoleOutput;
readonly PrimitiveManager primitiveManager;
readonly IDataStreamFactory streamFactory;
public MainForm()
public MainForm(PrimitiveManager primitiveManager)
: this(primitiveManager, primitiveManager) { }
public MainForm(PrimitiveManager primitiveManager, IDataStreamFactory streamFactory)
{
this.primitiveManager = primitiveManager;
this.streamFactory = streamFactory;
//Task.Run(async () => await _run());
InitializeComponent();
// myKQ = ConfigurationRequester.getApplicationQualifier();
@@ -109,11 +115,11 @@ namespace BCG
private IDictionary myGetLibraryData()
{
SortedList retVal = new SortedList();
string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames();
string[] primitiveNames = primitiveManager.getPrimitiveNames();
foreach (string pName in primitiveNames)
{
string aName = PrimitiveManager.instance.getAssemblyName(pName);
ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName);
string aName = primitiveManager.getAssemblyName(pName);
ICollection cNames = primitiveManager.getConstructorSigns(pName);
if (!retVal.Contains(aName))
retVal.Add(aName, new SortedList());
(retVal[aName] as IDictionary)[pName] = new ArrayList(cNames);
@@ -194,7 +200,7 @@ namespace BCG
myExp = exp;
myCrono.Restart();
(myCurrentStream = myExp.getView(new Dictionary<string, StatementExpression>(0))).onChange += myCallback;
(myCurrentStream = myExp.getView(new Dictionary<string, StatementExpression>(0), streamFactory)).onChange += myCallback;
myChangeStyle = true;
//new SimpleBindableStream(myExp.getView(new Hashtable())).onChange += new DataStream.ChangeDelegate(myCallback);
/*
@@ -432,7 +438,7 @@ namespace BCG
private void MainForm_Closed(object sender, System.EventArgs e)
{
if (myExp != null)
myExp.getView(new Dictionary<string, StatementExpression>(0)).onChange -= myCallback;
myExp.getView(new Dictionary<string, StatementExpression>(0), primitiveManager).onChange -= myCallback;
}
private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e)
@@ -667,7 +673,7 @@ namespace BCG
DialogResult dr = ofd.ShowDialog(this);
if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK)
{
PrimitiveManager.instance.loadAssembly(ofd.FileName);
primitiveManager.loadAssembly(ofd.FileName);
myLoadAssemblies();
}
}
@@ -4,6 +4,7 @@ using bts;
using bts.api.core;
using bts.md;
using Bts.DataStreamCore;
using Bts.DataStreamQueryAnalyzer;
using Bts.PubSub;
@@ -29,7 +30,14 @@ static class Program {
await Configuration.InitializeAsync(cfg.Sparkd);
PrimitiveManager.SetMarketDataClient(new MarketDataClient(new(TimeSpan.FromSeconds(0.5))));
Log.Gen.info("Starting DataStreamQueryAnalyzer");
var mainForm = new MainForm();
var primitiveManager = new PrimitiveManager(
new PrimitiveManagerConfig() {
Assemblies = new() {
"DataStream.dll",
"SharedStreams.dll"
},
});
var mainForm = new MainForm(primitiveManager);
if (cfg.StreamFile is not null) {
mainForm.loadFile(cfg.StreamFile);
}
@@ -1,13 +1,13 @@
namespace bts.BlueOrb {
using BCG;
using System;
using bts.md;
[DataStreamUserRespresentation("LiveFuturesBooktream")]
[DataStreamUserRespresentation(false)]
public class LiveFuturesBooktream : ViewStream {
public LiveFuturesBooktream(string future_target) {
ViewStream contract_str = new LiveFuturesContractStream(future_target);
ViewStream bbo_str = new BookStream();
public LiveFuturesBooktream(string future_target, MarketDataClient mdClient) {
ViewStream contract_str = new LiveFuturesContractStream(future_target, mdClient);
ViewStream bbo_str = new BookStream(mdClient);
ViewStream rs = new SimpleJoinStream(contract_str, "sec_key", bbo_str, "sec_key");
rs = new ProjectStream(rs, "left_sec:sec, left_sec_key:sec_key, bid_price, bid_sz:bid_size, ask_price, ask_sz:ask_size");
mySetRealSource(rs);
@@ -4,13 +4,14 @@
using BCG;
using bts.api;
using bts.md;
[DataStreamUserRespresentation("LiveFuturesContractStream")]
public class LiveFuturesContractStream : ViewStream {
public LiveFuturesContractStream(string target) {
public LiveFuturesContractStream(string target, MarketDataClient mdClient) {
var fc = new ContractUtils.FutureContract(target);
// temporary
ViewStream ins_str = new InstrumentsStream();
ViewStream ins_str = new InstrumentsStream(mdClient);
//new bts.BlueOrb.CSVReaderStream(@"C:\Temp\instruments.csv");
var rs = new LiveFuturesContractStream(ins_str, fc);
mySetRealSource(rs);
@@ -1,15 +1,16 @@
namespace bts.BlueOrb {
using bts.SilverPine;
using BCG;
using System;
using bts.md;
[DataStreamUserRespresentation("SpatialArbScoreStream")]
[DataStreamUserRespresentation(false)]
/// This stream is now created by <see cref="Bts.SharedStreams.StreamFactories.SpatialArbScoreStreamFactory"/>
public class SpatialArbScoreStream : ViewStream {
public SpatialArbScoreStream(int window_sz_secs, int sample_period_sz)
: this(TimeSpan.FromSeconds(window_sz_secs), TimeSpan.FromSeconds(sample_period_sz), ViewStreamDatabase.instance) { }
public SpatialArbScoreStream(MarketDataClient mdClient, int window_sz_secs, int sample_period_sz)
: this(mdClient, TimeSpan.FromSeconds(window_sz_secs), TimeSpan.FromSeconds(sample_period_sz), ViewStreamDatabase.instance) { }
public SpatialArbScoreStream(TimeSpan window_sz, TimeSpan sample_period, ViewStreamDatabase vsd) {
ViewStream rs = new SpatialArbStatsStream(window_sz, sample_period, vsd);
public SpatialArbScoreStream(MarketDataClient mdClient, TimeSpan window_sz, TimeSpan sample_period, ViewStreamDatabase vsd) {
ViewStream rs = new SpatialArbStatsStream(mdClient, window_sz, sample_period, vsd);
rs = new ThrottleStream(rs, TimeSpan.FromSeconds(1), vsd);
rs = new SortStream(rs, "-n_trade_ticks");
mySetRealSource(rs);
@@ -1,17 +1,15 @@
namespace bts.BlueOrb {
using BCG;
using SharpCompress.Common.Rar;
using bts.md;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
// For all the pairs generated by SpatialFuturePairsStream, this stream
// generates the statistics, by appending together a SpreadStatsStream
[DataStreamUserRespresentation("SpatialArbStatsStream")]
/// This stream is now created by <see cref="Bts.SharedStreams.StreamFactories.SpatialArbStatsStreamFactory"/>
[DataStreamUserRespresentation(false)]
public class SpatialArbStatsStream : ViewStream{
private ViewStream _spatial_pairs_stream;
private ViewStreamDatabase _vsd;
@@ -20,14 +18,14 @@
private ViewStream _bbo_stream;
private readonly TimeSpan _window_sz, _sampling_period_sz;
public SpatialArbStatsStream(int window_sz_seconds, int sampling_period_sz_seconds)
: this(TimeSpan.FromSeconds(window_sz_seconds), TimeSpan.FromSeconds(sampling_period_sz_seconds), ViewStreamDatabase.instance) { }
public SpatialArbStatsStream(MarketDataClient mdClient, int window_sz_seconds, int sampling_period_sz_seconds)
: this(mdClient, TimeSpan.FromSeconds(window_sz_seconds), TimeSpan.FromSeconds(sampling_period_sz_seconds), ViewStreamDatabase.instance) { }
public SpatialArbStatsStream(TimeSpan window_sz, TimeSpan sampling_period_sz, ViewStreamDatabase vsd) {
public SpatialArbStatsStream(MarketDataClient mdClient, TimeSpan window_sz, TimeSpan sampling_period_sz, ViewStreamDatabase vsd) {
_vsd = vsd;
_window_sz = window_sz;
_sampling_period_sz = sampling_period_sz;
_spatial_pairs_stream = new SpatialFuturePairsStream(vsd);
_spatial_pairs_stream = new SpatialFuturePairsStream(mdClient, vsd);
_bbo_stream = new BBOStream(vsd);
}
@@ -2,23 +2,22 @@
using BCG;
using bts.api;
using bts.md;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
[DataStreamUserRespresentation("SpatialFuturePairsStream")]
/// This stream is now created by <see cref="Bts.SharedStreams.StreamFactories.SpatialFuturePairsStreamFactory"/>
[DataStreamUserRespresentation(false)]
public class SpatialFuturePairsStream : ViewStream {
private ViewStream _main_stream;
private IBlueTableSchema _schema;
private HashSet<string> _key_map = new HashSet<string>();
public SpatialFuturePairsStream()
: this(ViewStreamDatabase.instance) { }
public SpatialFuturePairsStream(MarketDataClient mdClient)
: this(mdClient, ViewStreamDatabase.instance) { }
public SpatialFuturePairsStream(ViewStreamDatabase vsd) {
_main_stream = new InstrumentsStream(vsd);
public SpatialFuturePairsStream(MarketDataClient mdClient, ViewStreamDatabase vsd) {
_main_stream = new InstrumentsStream(mdClient, vsd);
_main_stream = new FilterStream(_main_stream, "sec_type='Future'");
_schema = new BlueTableSchema("SpatialFutures", new ColumnInfo("id", 0, TypeCode.Int32));
_schema = _schema.add_column(TypeCode.String, "s1");
@@ -18,7 +18,7 @@ namespace bts.BlueOrb{
using SecurityKey = System.UInt32;
using System.Diagnostics;
[DataStreamUserRespresentation("books")]
[DataStreamUserRespresentation(false)]
public class BookStream : ViewStream, IWorker {
private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
private DateTime _last_poll_time = DateTime.MinValue;
@@ -39,14 +39,13 @@ namespace bts.BlueOrb{
IDisposable? subscription;
// TODO: add cons for symbol, exchange etc
public BookStream()
: this(ViewStreamDatabase.instance) {
}
public BookStream(MarketDataClient mdClient)
: this(mdClient, ViewStreamDatabase.instance) { }
public BookStream(ViewStreamDatabase vsd) {
public BookStream(MarketDataClient mdClient, ViewStreamDatabase vsd) {
bts.Log.Gen.info("BookStream.cons");
_vsd = vsd;
_md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
_md_client = mdClient;
}
protected override void myOnFirstSubscriber() {
@@ -43,14 +43,13 @@ namespace bts.BlueOrb {
IDisposable? subscription;
// TODO: add cons for symbol, exchange etc
public InstrumentsStream()
: this(ViewStreamDatabase.instance) {
}
public InstrumentsStream(MarketDataClient mdClient)
: this(mdClient, ViewStreamDatabase.instance) { }
public InstrumentsStream(ViewStreamDatabase vsd) {
public InstrumentsStream(MarketDataClient mdClient, ViewStreamDatabase vsd) {
bts.Log.Gen.info("InstrumentsStream.cons");
_vsd = vsd;
_md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
_md_client = mdClient;
}
protected override void myOnFirstSubscriber() {
@@ -213,7 +213,14 @@ namespace bts.BlueOrb {
}
internal OHLCVStream(ViewStream source, OHLCVBarSize bar_size) {
ViewStream rs = source;
rs = new SchemaGuesserStream(rs);
if (bar_size == OHLCVBarSize.ONE_DAY) {
rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
}
rs = new NewProjectStream(rs, "symbol:contract");
mySetRealSource(rs);
}
}
}
@@ -1,19 +1,23 @@
namespace bts.BlueOrb {
using BCG;
using Bts.DataStreamCore;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
[DataStreamUserRespresentation("nl")]
[DataStreamUserRespresentation(false)]
public class BlueOrbNLStream : ViewStream {
private readonly string _nl_prompt;
readonly IDataStreamFactory streamFactory;
private ViewStream _nl_to_dsl_stream, _dsl_stream;
private IDictionary<string, object> _dp_metadata = new Dictionary<string, object>();
public BlueOrbNLStream(string nl_prompt) {
public BlueOrbNLStream(string nl_prompt, IDataStreamFactory streamFactory) {
_nl_prompt = nl_prompt;
this.streamFactory = streamFactory;
}
protected override void myOnFirstSubscriber() {
@@ -27,7 +31,7 @@
string dsl = r.get<string>("response");
_dp_metadata["dsl"] = dsl;
Log.Gen.info($"NLStream._handle_dsl_callback dsl: {dsl}");
_dsl_stream = new DSLStream(dsl);
_dsl_stream = new DSLStream(dsl, streamFactory);
_dsl_stream.onChange += _handle_data_cb;
}
@@ -0,0 +1,77 @@
namespace bts.SilverPine;
using Spark.Core.Json;
using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;
public class NLPromptConfig {
static NLPromptConfig? instance;
public static NLPromptConfig Instance =>
instance ?? throw new InvalidOperationException(
"SetGlobalConfig must be called to initialize config static singleton");
public List<FewShotPrompt> prompts { get; } = new(0);
public static void SetGlobalConfig(NLPromptConfig cfg) {
// This should only be called once
Log.isnull(instance);
instance = Log.notnull(cfg);
}
public static NLPromptConfig Default() => new();
// Represents a pair of a natural language query and its dsl
public class FewShotExample {
public string? natural_language { get; set; }
public string? dsl { get; set; }
}
public class FewShotPrompt {
public string? category { get; set; }
public string? description { get; set; }
public bool IsEnabled { get; set; } = true;
// MultiLineDescription is simply here for convenience. It is functionally redundant, as it
// serves the same purpose as 'description'. If 'description' is too long, you can break it
// up into an array of strings and assign it to MultiLineDescription instead.
public List<string>? MultiLineDescription { get; set; }
// the actual prompt given by the user/CME
public string? prompt { get; set; }
public List<FewShotExample> examples { get; } = new(0);
public override string ToString() {
var desc = description ?? string.Concat(MultiLineDescription.empty_if_null());
string rv = $"System: {desc}\n";
int ex_no = 0;
foreach (FewShotExample ex in examples ?? new(0)) {
rv += $"{++ex_no}. Natural Language: \"{ex.natural_language}\"" + "\n";
rv += $" DSL: {ex.dsl}\n";
}
return rv;
}
// Reads normal strings and arrays of lines
public class FewShotPromptLinesConverter : JsonConverter<string> {
public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
if (reader.ReadString(out var str)) {
return str;
}
if (reader.ReadArray(out var array)) {
return string.Join('\n', array.Select(v => v?.AsString()));
}
throw new JsonException("Failed to parse FewShotPrompt lines");
}
public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) =>
writer.WriteStringValue(value);
}
}
}
@@ -0,0 +1,44 @@
namespace bts.SilverPine;
using BCG;
using bts.BlueOrb;
using Bts.DataStreamCore;
using mbts.utils;
using System.Collections.Generic;
public class NLStreamFactory : IRecursiveDataStreamFactory {
IDataStreamFactory? compositeFactory;
public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) {
if (SupportedStreams.NaturalLanguage.IsMatch(streamId)) {
if (compositeFactory is null) {
Log.DataFlow.warn("Failed to create NLStream because 'compositeFactory' is null.");
return null;
}
if (args.SingleOrDefault().Value is string prompt) {
return new BlueOrbNLStream(prompt, compositeFactory);
}
}
return null;
}
public IEnumerable<IDataStreamMetadata> GetSupportedStreams() {
yield return SupportedStreams.NaturalLanguage;
}
public void OnCompositeFactory(IDataStreamFactory compositeFactory) =>
this.compositeFactory = compositeFactory;
static class SupportedStreams {
public static readonly DataStreamMetadata NaturalLanguage = new() {
Name = "nl",
AlternateNames = new() { "naturalLanguage" }
};
public static string NlName => ExceptionAssert.not_null(NaturalLanguage.Name);
}
}
@@ -0,0 +1,28 @@
namespace Bts.SharedStreams.Services;
using bts.SilverPine;
using Bts.DataStreamCore;
using Bts.SharedStreams.StreamFactories;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
public static class SharedStreamsServices {
public static IServiceCollection AddMarketDataStreams(this IServiceCollection services) =>
services.addStreamFactory<BookStreamFactory>()
.addStreamFactory<InstrumentStreamFactory>();
public static IServiceCollection AddNaturalLanguageStreams(this IServiceCollection services, IConfigurationSection configurationSection) {
services.Configure<NLPromptConfig>(configurationSection);
NLPromptConfig.SetGlobalConfig(configurationSection.Get<NLPromptConfig>() ?? NLPromptConfig.Default());
// TODO: integrate config into a NLStream provider (so it can provide config values
// instead of the ViewStream's getting them from a static singleton)
return services.AddSingleton<NLStreamFactory>()
.AddSingleton<IDataStreamFactory, NLStreamFactory>(sp => sp.GetRequiredService<NLStreamFactory>());
}
static IServiceCollection addStreamFactory<T>(this IServiceCollection services) where T : class, IDataStreamFactory =>
services.AddSingleton<T>()
.AddSingleton<IDataStreamFactory, T>(sp => sp.GetRequiredService<T>());
}
@@ -0,0 +1,32 @@
namespace Bts.SharedStreams.StreamFactories;
using BCG;
using bts.md;
using Bts.DataStream;
using Bts.DataStreamCore;
using Bts.SharedStreams.MarketData;
using Microsoft.Extensions.Logging;
using System;
public class BookStreamFactory : DataStreamFactory {
public BookStreamFactory(MarketDataClient? mdClient, ILogger<BookStreamFactory> logger) {
this.Add<BookStreamParams>("book", CreateBook, logger);
this.Add<BookStreamParams>(new[] { "bbo", "LiveBBOStream" }, CreateBbo, logger);
ViewStream? CreateBook(BookStreamParams streamParams) =>
mdClient?.books(streamParams.GetSecurityFilter())
?.AsDataPackets(streamParams.GetSchemaBuilder().BookTupleAdapter())
?.AsViewStream()
?? throw new InvalidOperationException("Live md not available");
ViewStream? CreateBbo(BookStreamParams streamParams) =>
mdClient?.books(streamParams.GetSecurityFilter())
?.AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter())
?.AsViewStream()
?? throw new InvalidOperationException("Live md not available");
}
}
@@ -0,0 +1,78 @@
namespace Bts.SharedStreams.StreamFactories;
using BCG;
using bts.api;
using Bts.DataStreamCore;
public class InstrumentSchemaBuilder {
public bool ShowVenue { get; set; } = true;
public bool ShowAsset { get; set; } = true;
public bool ShowType { get; set; } = true;
public bool ShowName { get; set; } = true;
public bool ShowId { get; set; } = true;
public bool ShowCashPerPoint { get; set; } = true;
public bool ShowLegId { get; set; } = true;
public bool ShowLegVenue { get; set; }
public bool ShowLegAsset { get; set; }
public bool ShowLegType { get; set; }
public bool ShowLegName { get; set; } = true;
public bool ShowLegCashPerPoint { get; set; }
public int NumLegs { get; set; } = 2;
public string TableName { get; set; } = "Instrument";
public BlueTableSchemaAdapter<SecurityImmutable> SecurityAdapter() {
var adapter = new BlueTableSchemaAdapter<SecurityImmutable>(TableName);
adapter.AddColumn("id", r => (int)r.Id);
if (ShowVenue) {
adapter.AddColumn(WellKnownColumns.Venue.Name, b => b.Exchange.ToString());
}
if (ShowType) {
adapter.AddColumn(WellKnownColumns.SecurityType.Name, b => b.security_type().ToString());
}
if (ShowAsset) {
adapter.AddColumn(WellKnownColumns.Asset.Name, b => b.Symbol);
}
if (ShowName) {
adapter.AddColumn(WellKnownColumns.SecurityName.Name, b => b.ToString());
}
if (ShowId) {
adapter.AddColumn(WellKnownColumns.SecurityId.Name, b => b.Id);
}
if (ShowCashPerPoint) {
adapter.AddColumn(WellKnownColumns.CashPerPoint.Name, b => b.ContractSpec.CashPerPoint);
}
for (int i = 0; i < NumLegs; i++) {
if (ShowLegVenue) {
adapter.AddColumn(WellKnownColumns.LegVenue(i).Name, Get(l => l.Security.Exchange.ToString(), null));
}
if (ShowLegType) {
adapter.AddColumn(WellKnownColumns.LegSecurityType(i).Name, Get(l => l.Security.security_type().ToString(), null));
}
if (ShowLegAsset) {
adapter.AddColumn(WellKnownColumns.LegAsset(i).Name, Get(l => l.Security.Symbol, null));
}
if (ShowLegName) {
adapter.AddColumn(WellKnownColumns.LegSecurityName(i).Name, Get(l => l.Security.ToString(), null));
}
if (ShowLegId) {
adapter.AddColumn(WellKnownColumns.LegSecurityId(i).Name, Get(l => (int)l.Security.Id, 0));
}
if (ShowLegCashPerPoint) {
adapter.AddColumn(WellKnownColumns.LegCashPerPoint(i).Name, Get(l => l.Security.ContractSpec.CashPerPoint, double.NaN));
}
Func<SecurityImmutable, T> Get<T>(Func<ContractLegImmutable, T> getValue, T defaultValue) =>
s => {
ContractLegImmutable? leg = s.ContractCase switch {
Security.ContractOneofCase.Spread => s.Spread.Legs.Skip(i).TryGetFirst(out var l) ? l : null,
Security.ContractOneofCase.Synthetic => s.Synthetic.Legs.Skip(i).TryGetFirst(out var l) ? l : null,
_ => null,
};
return leg.HasValue ? getValue(leg.Value) : defaultValue;
};
}
return adapter;
}
}
@@ -0,0 +1,34 @@
namespace Bts.SharedStreams.StreamFactories;
using BCG;
using bts.api.core;
using bts.BlueOrb;
using bts.md;
using Bts.DataStream;
using Bts.DataStreamCore;
using Bts.SharedStreams.MarketData;
using Microsoft.Extensions.Logging;
class InstrumentStreamFactory : DataStreamFactory {
public InstrumentStreamFactory(MarketDataClient mdClient, ILogger<InstrumentStreamFactory> logger) {
this.Add<InstrumentStreamParams>("instrument", CreateInstrument, logger);
this.Add<LiveFutureContractStreamParams>(new[] { "liveFutures", "liveFuturesContract", "LiveFuturesContractStream" }, CreateLiveFutureContract, logger);
ViewStream? CreateInstrument(InstrumentStreamParams streamParams) =>
mdClient.securities(streamParams.GetSecurityCriteria())
.Table(s => s.Id)
.AsDataPackets(streamParams.GetSchemaBuilder().SecurityAdapter())
.AsViewStream();
ViewStream? CreateLiveFutureContract(LiveFutureContractStreamParams streamParams) =>
new LiveFuturesContractStream(streamParams.Target ?? string.Empty, mdClient);
}
public class LiveFutureContractStreamParams {
[StreamArgument(RegexNames = new[] { "^(symbol|sym|product|asset|target|tgt)$" })]
public string? Target { get; set; }
}
}
@@ -0,0 +1,82 @@
namespace Bts.SharedStreams.StreamFactories;
using bts.api;
using Bts.DataStreamCore;
public class InstrumentStreamParams {
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.VenueRegex })]
public Venue? Venue { get; set; }
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.AssetRegex })]
public string? Asset { get; set; }
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityType })]
public SecurityType? Type { get; set; }
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.SecurityName })]
public string? Name { get; set; }
[StreamArgument("legs", "nLegs", "numLegs")]
public int NumLegs { get; set; } = 5;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowVenueRegex })]
public bool ShowVenue { get; set; } = true;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowAssetRegex })]
public bool ShowAsset { get; set; } = true;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowSecurityTypeRegex })]
public bool ShowType { get; set; } = true;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowSecurityNameRegex })]
public bool ShowName { get; set; } = true;
public bool ShowId { get; set; } = false;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowCashPerPointRegex })]
public bool ShowCashPerPoint { get; set; } = true;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegVenueRegex })]
public bool ShowLegVenue { get; set; } = true;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegAssetRegex })]
public bool ShowLegAsset { get; set; } = true;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegSecurityTypeRegex })]
public bool ShowLegType { get; set; } = true;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegNameRegex })]
public bool ShowLegName { get; set; } = true;
public bool ShowLegId { get; set; } = false;
[StreamArgument(RegexNames = new[] { WellKnownStreamArgumentAliases.ShowLegCashPerPointRegex })]
public bool ShowLegCashPerPoint { get; set; } = true;
public mbts.spark.SecurityCriteria GetSecurityCriteria() {
var clause = new mbts.spark.SecurityCriteriaClause(
exchanges: Venue.HasValue ? new[] { Venue.Value } : null,
security_types: Type.HasValue ? new[] { Type.Value } : null,
symbols: !string.IsNullOrEmpty(Asset) ? new[] { Asset } : null,
names: !string.IsNullOrEmpty(Name) ? new[] { Name } : null);
return clause.is_defined ? new(new[] { clause }) : new();
}
public InstrumentSchemaBuilder GetSchemaBuilder() =>
new() {
NumLegs = NumLegs,
ShowVenue = ShowVenue,
ShowType = ShowType,
ShowAsset = ShowAsset,
ShowName = ShowName,
ShowId = ShowId,
ShowCashPerPoint = ShowCashPerPoint,
ShowLegVenue = ShowLegVenue,
ShowLegType = ShowLegType,
ShowLegAsset = ShowLegAsset,
ShowLegName = ShowLegName,
ShowLegId = ShowLegId,
ShowLegCashPerPoint = ShowLegCashPerPoint,
};
}
@@ -3,8 +3,10 @@ namespace Spark.Api.MarketData {
using bts.api;
using bts.api.domain;
using bts.md;
using bts.utils;
using System;
using System.Linq;
public readonly struct BookTuple : IEquatable<BookTuple>, IJsonFormatter<BookTuple> {
readonly TickDisplaySettingsImmutable? _quantity_display;
@@ -19,9 +21,13 @@ namespace Spark.Api.MarketData {
_quantity_display = quantity_display;
}
public static BookTuple Empty(SecurityImmutable security) =>
new(security, Book.empty(security.Id));
public SecurityImmutable security { get; }
public uint security_id => security.Id;
public Book book { get; }
public Bbo bbo {
get {
(double bidp, double bidq) = book.bids.Count > 0 ? (book.bids[0].price, book.bids[0].size) : (double.NaN, double.NaN);
@@ -33,6 +39,8 @@ namespace Spark.Api.MarketData {
public TickDisplaySettingsImmutable quantity_display =>
_quantity_display ?? Security.Defaults.quantity_display_settingsi;
public DateTime? Timestamp { get; init; }
public bool Equals(BookTuple other) =>
security.Equals(other.security) &&
book.Equals(other.book);
@@ -52,4 +60,15 @@ namespace Spark.Api.MarketData {
public static bool operator !=(BookTuple left, BookTuple right) =>
!(left == right);
}
public static class BookTupleExtensions {
public static Book NotionalBook(this BookTuple bookTuple) =>
bookTuple.book.ToNotional(bookTuple.security.ContractSpec.CashPerPoint);
public static Book ToNotional(this Book book, double cpp) =>
new(
book.security_id,
book.bids.Select(b => new PriceSize((float)(b.size * cpp * b.price), b.price)).ToList(),
book.asks.Select(a => new PriceSize((float)(a.size * cpp * a.price), a.price)).ToList());
}
}
@@ -60,7 +60,7 @@ namespace bts.md {
}
public IObservable<TableUpdate<BookTuple, uint>> books(SecurityCriteria? criteria = null) =>
books(criteria is null ? null :
books(criteria is null || !criteria.is_defined() ? null :
new Func<SecurityImmutable, bool>(s => criteria.includes(s)));
public IObservable<TableUpdate<BookTuple, uint>> books(Func<SecurityImmutable, bool>? filter) {
+2 -4
View File
@@ -2,14 +2,12 @@ namespace bts.md {
using System;
using Size = System.Single;
public class PriceSize
{
public class PriceSize {
public Size size { get; }
public Size direct_size { get; }
public Size implied_size { get; }
public double price { get; }
public PriceSize(Size sz, Size direct_sz, Size implied_sz, double prx)
{
public PriceSize(Size sz, Size direct_sz, Size implied_sz, double prx) {
size = sz;
direct_size = direct_sz;
implied_size = implied_sz;
@@ -0,0 +1,26 @@
#nullable enable
using bts.api;
using Spark.Api.MarketData;
using System.Collections.Generic;
public readonly struct SyntheticBook {
public SyntheticBook(
uint securityId, IReadOnlyList<SyntheticBookLevel> bids, IReadOnlyList<SyntheticBookLevel> asks) {
SecurityId = securityId;
Bids = bids;
Asks = asks;
}
public uint SecurityId { get; }
public IReadOnlyList<SyntheticBookLevel> Bids { get; }
public IReadOnlyList<SyntheticBookLevel> Asks { get; }
public SyntheticBookLevel this[Side side, int level] =>
side switch {
Side.Buy => level < Bids.Count ? Bids[level] : new(double.NaN, double.NaN, double.NaN),
Side.Sell => level < Asks.Count ? Asks[level] : new(double.NaN, double.NaN, double.NaN),
_ => throw new KeyNotFoundException("Invalid side"),
};
}
@@ -3,6 +3,7 @@ namespace Spark.Api.MarketData;
using bts;
using bts.api;
using bts.collections;
using bts.md;
using bts.utils;
@@ -18,7 +19,7 @@ public static class SyntheticBookCalculator {
GetSyntheticDepth(legDepths.Select(GetBidLegParams).ToList()).Select(ToPriceSize).ToList(),
GetSyntheticDepth(legDepths.Select(GetAskLegParams).ToList()).Select(ToPriceSize).ToList());
PriceSize ToPriceSize(DepthLevel level) =>
static PriceSize ToPriceSize(DepthLevel level) =>
new((float)level.Quantity, level.Price);
(double Quantity, double TickSize, Book Book) GetBidLegParams((ContractLegImmutable, Book) ld) =>
@@ -28,6 +29,22 @@ public static class SyntheticBookCalculator {
(-ld.Item1.signed_quantity(), ld.Item1.Security.ContractSpec.LowTick, ld.Item2);
}
public static Book GetSyntheticBook(uint securityId, IEnumerable<(double Quantity, double TickSize, Book Book)> legDepths) {
return new(
securityId,
GetSyntheticDepth(legDepths.ToList()).Select(ToPriceSize).ToList(),
GetSyntheticDepth(legDepths.Select(l => (-l.Quantity, l.TickSize, l.Book)).ToList()).Select(ToPriceSize).ToList());
static PriceSize ToPriceSize(DepthLevel level) =>
new((float)level.Quantity, level.Price);
}
public static SyntheticBook GetSyntheticNotionalBook(uint securityId, IEnumerable<(double Quantity, double TickSize, double CashPerPoint, Book Book)> legDepths) =>
new(
securityId,
GetSyntheticNotionalDepth(legDepths.ToList()).ToList(),
GetSyntheticNotionalDepth(legDepths.Select(l => (-l.Quantity, l.TickSize, l.CashPerPoint, l.Book)).ToList()).ToList());
public static List<DepthLevel> GetSyntheticDepth(IEnumerable<(ContractLegImmutable, Book)> legDepths) {
return GetSyntheticDepth(
legDepths.Select(getLegDepthConsumer).ToList());
@@ -37,15 +54,18 @@ public static class SyntheticBookCalculator {
Log.assert(leg.Quantity.is_real(), "Leg quantity must be real");
Log.assert(leg.Security.HasContractSpec && leg.Security.ContractSpec.LowTick.is_real_positive(),
"Leg security LowTick must be real and positive");
return LegDepthConsumer.Create(leg.Quantity, leg.Security.ContractSpec.LowTick);
var depthLevels = (leg.Side == Side.Buy ? book.bids : book.asks)
.Select(lvl => new DepthLevel(lvl.price, lvl.size))
.ToArray();
return LegDepthConsumer.Create(leg.Quantity, leg.Security.ContractSpec.LowTick, depthLevels);
}
}
public static List<DepthLevel> GetSyntheticDepth(IEnumerable<(double Quantity, double TickSize, Book Book)> legDepths) {
return GetSyntheticDepth(
legDepths.Select(getLegDepthConsumer).ToList());
legDepths.Select(GetLegDepthConsumer).ToList());
LegDepthConsumer getLegDepthConsumer((double Quantity, double TickSize, Book Book) legDepth) {
LegDepthConsumer GetLegDepthConsumer((double Quantity, double TickSize, Book Book) legDepth) {
var (qty, tickSz, bk) = legDepth;
Log.assert(qty.is_real(), "Leg quantity must be real");
Log.assert(tickSz.is_real_positive(),
@@ -54,6 +74,29 @@ public static class SyntheticBookCalculator {
}
}
public static List<SyntheticBookLevel> GetSyntheticNotionalDepth(IEnumerable<(double Quantity, double TickSize, double CashPerPoint, Book Book)> legDepths) {
var legs = legDepths.Select(InitializeLegDepth).ToList();
if (legs.Count == 0) {
return new(0);
}
return legs.Skip(1).Aggregate(legs[0], AggregateLegDepth).Depth.Levels;
SyntheticLegDepthConsumer InitializeLegDepth((double Quantity, double TickSize, double CashPerPoint, Book Book) legDepth) {
var (qty, tickSz, cashPerPoint, bk) = legDepth;
Log.assert(qty.is_real(), "Leg quantity must be real");
Log.assert(tickSz.is_real_positive(),
"Leg tickSz must be real and positive");
var levels = (qty > 0 ? bk.bids : bk.asks).ToArray(l => new SyntheticBookLevel(l.price, l.size, l.NotionalValue(cashPerPoint)));
return SyntheticLegDepthConsumer.Create(qty, tickSz, levels);
}
SyntheticLegDepthConsumer AggregateLegDepth(SyntheticLegDepthConsumer l1, SyntheticLegDepthConsumer l2) =>
new(new(getSyntheticNotionalDepth(l1, l2).ToList())) {
LegQuantity = 1,
QuantityTick = MathUtils.lcm(l1.QuantityTick, l2.QuantityTick),
};
}
public static List<DepthLevel> GetSyntheticDepth(List<LegDepthConsumer> legs) {
if (legs.Count == 0) {
return new(0);
@@ -90,6 +133,39 @@ public static class SyntheticBookCalculator {
}
}
static IEnumerable<SyntheticBookLevel> getSyntheticNotionalDepth(
SyntheticLegDepthConsumer leg1, SyntheticLegDepthConsumer leg2) {
while (!leg1.Depth.IsDone && !leg2.Depth.IsDone) {
if (!leg1.LegQuantity.is_real() || !leg2.LegQuantity.is_real()) {
throw new ArgumentException("Invalid leg quantities");
}
double leg2PerLeg1 = (leg2.LegQuantity / leg1.LegQuantity).abs();
double leg1PerLeg2 = (leg1.LegQuantity / leg2.LegQuantity).abs();
double avail1 = leg1.Depth.AvailableQuantity(1);
double avail2 = leg2.Depth.AvailableQuantity(1);
(double take1, double take2) = (avail1 / leg1PerLeg2) < (avail2 / leg2PerLeg1)
? (avail1, avail1 * leg2PerLeg1)
: (avail2 * leg1PerLeg2, avail2);
var (qty1, cost1, notional1) = leg1.Depth.ConsumeQuantity(take1);
var (qty2, cost2, notional2) = leg2.Depth.ConsumeQuantity(take2);
Log.assert(qty2 / qty1 == leg2PerLeg1);
var price1 = cost1 / qty1;
var price2 = cost2 / qty2;
price1 *= leg1.LegQuantity;
price2 *= leg2.LegQuantity;
if (leg1.LegQuantity < 0) {
notional1 = -notional1;
}
if (leg2.LegQuantity < 0) {
notional2 = -notional2;
}
yield return new(
Price: price1 + price2,
Quantity: qty1,
NotionalValue: notional1 + notional2);
}
}
public record DepthLevel(double Price, double Quantity) {
public override string ToString() {
return $"{Price} ({Quantity})";
@@ -173,6 +249,7 @@ public static class SyntheticBookCalculator {
public double LegQuantity { get; set; } = 1;
public double QuantityTick { get; set; } = 1;
public double CashPerPoint { get; set; } = 1;
public DepthConsumer Depth { get; } = new();
public static LegDepthConsumer Create(double legQuantity, double quantityTick, params DepthLevel[] levels) =>
@@ -0,0 +1,12 @@
#nullable enable
namespace Spark.Api.MarketData;
/// <summary>
/// Like a normal BookLevel, but includes the NotionalValue, as that could only
/// otherwise be computed with the leg quantities and cash per point.
/// </summary>
/// <param name="Price"></param>
/// <param name="Quantity"></param>
/// <param name="NotionalValue"></param>
public readonly record struct SyntheticBookLevel(double Price, double Quantity, double NotionalValue);
@@ -0,0 +1,73 @@
#nullable enable
namespace Spark.Api.MarketData;
using System;
using System.Collections.Generic;
using System.Linq;
public class SyntheticDepthConsumer {
public SyntheticDepthConsumer() { }
public SyntheticDepthConsumer(params SyntheticBookLevel[] levels) : this(levels.AsEnumerable()) { }
public SyntheticDepthConsumer(IEnumerable<SyntheticBookLevel> levels) =>
Levels.AddRange(levels);
public int Pos { get; private set; } = 0;
public double CurrLevelConsumed { get; private set; } = 0;
public List<SyntheticBookLevel> Levels { get; } = new();
public bool IsDone =>
Pos == Levels.Count || (Pos == Levels.Count - 1 && Levels.Last().Quantity == CurrLevelConsumed);
public double AvailableQuantity(int levelLimit) {
int end = Math.Min(Pos + levelLimit, Levels.Count);
double rv = 0;
for (int i = Pos; i < end; i++) {
double levelQty = Levels[i].Quantity;
if (i == Pos) {
levelQty -= CurrLevelConsumed;
}
rv += levelQty;
}
return rv;
}
public (double Qty, double Price, double Notional) ConsumeQuantity(double quantity) {
double consumed = 0;
double consumedNotional = 0;
double price = 0;
for (; Pos < Levels.Count; Pos++) {
var level = Levels[Pos];
double levelQty = level.Quantity - CurrLevelConsumed;
var rem = quantity - consumed;
if (levelQty > rem) {
CurrLevelConsumed += rem;
consumed = quantity;
var remCost = rem * Levels[Pos].Price;
price += remCost;
consumedNotional += Levels[Pos].NotionalValue * rem / Levels[Pos].Quantity;
break;
}
consumed += levelQty;
price += levelQty * Levels[Pos].Price;
consumedNotional += Levels[Pos].NotionalValue * levelQty / Levels[Pos].Quantity;
CurrLevelConsumed = 0;
}
return (consumed, price, consumedNotional);
}
}
public class SyntheticLegDepthConsumer {
public SyntheticLegDepthConsumer() : this(new()) { }
public SyntheticLegDepthConsumer(SyntheticDepthConsumer depth) =>
Depth = depth;
public double LegQuantity { get; set; } = 1;
public double QuantityTick { get; set; } = 1;
public SyntheticDepthConsumer Depth { get; } = new();
public static SyntheticLegDepthConsumer Create(double legQuantity, double quantityTick, params SyntheticBookLevel[] levels) =>
new(new(levels)) {
LegQuantity = legQuantity,
QuantityTick = quantityTick,
};
}
@@ -108,10 +108,19 @@ namespace mbts.spark {
security_criteria.is_defined() &&
security_criteria.clauses.empty_if_null().Any(c => c.includes(security));
public static bool includes(
this SecurityCriteria security_criteria, (Venue? Venue, SecurityType? Type, string? Symbol) tup) =>
security_criteria.is_defined() &&
security_criteria.clauses.empty_if_null().Any(c => c.includes(tup));
public static bool includes(
this SecurityCriteriaClause clause, SecurityImmutable security) =>
_to_predicates(clause).AnyAndAll(p => p(security));
public static bool includes(
this SecurityCriteriaClause clause, (Venue? Venue, SecurityType? Type, string? Symbol) tup) =>
_to_symbol_predicates(clause).AnyAndAll(p => p(tup));
public static bool is_subset_of(
this SecurityCriteriaClause clause, SecurityCriteriaClause other) =>
// exchanges
@@ -178,6 +187,10 @@ namespace mbts.spark {
)
);
public static bool includes(this SecurityCriteria security_criteria, (Venue Venue, SecurityType Type, string? symbol) sec_tup) =>
security_criteria.is_defined() &&
security_criteria.clauses.empty_if_null().Any(c => c.includes(sec_tup));
public static SecurityCriteriaClause with(this SecurityCriteriaClause clause,
Venue[]? exchanges = null,
SecurityType[]? security_types = null,
@@ -472,5 +485,17 @@ namespace mbts.spark {
yield return sec => clause.names.Contains(sec.ToString());
}
}
static IEnumerable<Func<(Venue? Venue, SecurityType? Type, string? Symbol), bool>> _to_symbol_predicates(SecurityCriteriaClause clause) {
if (!clause.exchanges.is_null_or_empty()) {
yield return tup => !tup.Venue.HasValue || clause.exchanges.Contains(tup.Venue.Value);
}
if (!clause.symbols.is_null_or_empty()) {
yield return tup => tup.Symbol is null || clause.symbols.Contains(tup.Symbol);
}
if (!clause.security_types.is_null_or_empty()) {
yield return tup => !tup.Type.HasValue || clause.security_types.Contains(tup.Type.Value);
}
}
}
}
@@ -0,0 +1,9 @@
#nullable enable
namespace Spark.Core.Json;
using System.Text.Json.Serialization;
[JsonConverter(typeof(JsonMultiLineStringConverter))]
public class JsonMultiLineString {
public string? Value { get; set; }
}
@@ -0,0 +1,39 @@
#nullable enable
namespace Spark.Core.Json;
using System;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
// Reads normal strings and arrays of lines
public class JsonMultiLineStringConverter : JsonConverter<JsonMultiLineString> {
public override JsonMultiLineString? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
if (reader.ReadString(out var str)) {
return new() { Value = str };
}
if (reader.ReadArray(out var array)) {
return new() { Value = string.Join('\n', array.Select(v => v?.AsString())) };
}
throw new JsonException("Failed to parse multi-line string");
}
public override void Write(Utf8JsonWriter writer, JsonMultiLineString value, JsonSerializerOptions options) =>
writer.WriteStringValue(value.Value);
}
public class JsonMultiLineStringValueConverter : JsonConverter<string> {
public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
if (reader.ReadString(out var str)) {
return str;
}
if (reader.ReadArray(out var array)) {
return str;
}
throw new JsonException("Failed to parse multi-line string");
}
public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) =>
writer.WriteStringValue(value);
}
@@ -3,6 +3,7 @@ namespace bts.api.core {
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
public class TableUpdate<T, TKey> where TKey : IEquatable<TKey> {
@@ -277,6 +278,17 @@ namespace bts.api.core {
_removed[kv.Key] = kv.Value;
}
public bool TryGetValue(TKey key, [MaybeNullWhen(false)]out T result) {
if(_added.TryGetValue(key, out result)) {
return true;
}
if (_changes.TryGetValue(key, out var ch)) {
result = ch.New;
return true;
}
return !_removed.ContainsKey(key) && _state.state.TryGetValue(key, out result);
}
/// <summary>
/// Creates a TableUpdate instance from the current state
/// </summary>
@@ -68,6 +68,9 @@ namespace mbts.ui.utility {
return Activator.CreateInstance(enum_type);
}
public static T? TryParse<T>(string value, bool ignoreCase = true) where T : struct, Enum =>
Enum.TryParse(value, ignoreCase, out T res) ? res : null;
public static T try_convert<T>(object value) {
var numeric_type = Enum.GetUnderlyingType(typeof(T));
var numeric_value = Convert.ChangeType(value, numeric_type);
@@ -16,6 +16,7 @@ namespace bts.websocket_core {
using bts.api.position;
using bts.api.synthetic;
using Spark.Api.Services;
using Bts.DataStreamCore;
/// <summary>
@@ -26,10 +27,12 @@ namespace bts.websocket_core {
readonly AdminService adminService;
readonly Spark.Api.IServiceProvider _service_ctx;
readonly IUserDataPermissions userDataPermissions;
readonly IDataStreamFactory streamFactory;
public WebsocketDataSync(Spark.Api.IServiceProvider service_ctx, IUserDataPermissions userDataPermissions) {
public WebsocketDataSync(Spark.Api.IServiceProvider service_ctx, IUserDataPermissions userDataPermissions, IDataStreamFactory streamFactory) {
_service_ctx = service_ctx;
this.userDataPermissions = userDataPermissions;
this.streamFactory = streamFactory;
adminService = service_ctx.GetService<AdminService>();
}
@@ -328,6 +331,7 @@ namespace bts.websocket_core {
req,
Md_Client,
Orders_Client,
streamFactory,
obj => send_message(obj, req.client_id),
out hdlr,
out status)) {
@@ -224,6 +224,7 @@ namespace bts.websocket_core {
}
static bool TryGetDataStream(
WebsocketRequest req,
IDataStreamFactory streamFactory,
[NotNullWhen(true)] out ViewStream? stream,
[NotNullWhen(false)] out string error
) {
@@ -253,7 +254,7 @@ namespace bts.websocket_core {
}
Exception? exception = null;
try {
stream = exp.getView(new Dictionary<string, StatementExpression>(0));
stream = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
} catch (Exception ex) {
exception = ex;
}
@@ -306,6 +307,7 @@ namespace bts.websocket_core {
WebsocketRequest req,
MarketDataClient md,
OrdersClient oc,
IDataStreamFactory streamFactory,
Action<Object> sm,
[NotNullWhen(true)] out WebsocketSubscriptionHandler? hdlr,
out string status
@@ -372,7 +374,7 @@ namespace bts.websocket_core {
}
case SUBSCRIPTION_TYPE.DATA_STREAM: {
// TODO: make this nullable
if (!TryGetDataStream(req, out var filter, out status)) {
if (!TryGetDataStream(req, streamFactory, out var filter, out status)) {
return false;
}
sub = GetDataStreamSubscriber(filter, id.Value.Val);