feat: Add BboPriceSummaryStream for Brandon Chen

This commit is contained in:
John Colvin committed 2025-11-15 12:45:08 -05:00
1 parent c365330b08
commit 4addb58743
12 files changed
+1030 -12

No files matched your search

@@ -0,0 +1,19 @@
namespace bts.BlueOrb;
using BCG;
public class ColumnMaskRow : IBlueRow {
readonly IBlueRow inputRow;
readonly (string? Name, int Index)[] columnMap;
public ColumnMaskRow(IBlueRow inputRow, (string? Name, int Index)[] columnMap, IBlueTableSchema schema) {
this.inputRow = inputRow;
this.columnMap = columnMap;
this.schema = schema;
}
public IBlueTableSchema schema { get; }
public object? this[int index] =>
inputRow[columnMap[index].Index];
}
@@ -0,0 +1,68 @@
namespace bts.BlueOrb;
using BCG;
using Bts.DataStream;
using Spark.Core.Disposables;
using System;
using System.Diagnostics.CodeAnalysis;
public class ColumnMaskStream : ViewStream {
readonly ViewStream source;
readonly string? columnMaskExpression;
IDisposable? sub;
IBlueTableSchema? outputSchema;
// Each item in this array represents an output column as a tuple of
// new name, and corresponding index in the input schema.
(string? OutputName, int InputIndex)[]? columnMappings;
(string From, string To)[]? columnNameMappings;
readonly bool includeUnmappedColumns;
public ColumnMaskStream(ViewStream source, string columnMaskExpression) {
this.source = source;
this.columnMaskExpression = columnMaskExpression;
}
public ColumnMaskStream(ViewStream source, bool includeUnmappedColumns, params (string From, string To)[] columnNameMappings) {
this.source = source;
this.columnNameMappings = columnNameMappings;
this.includeUnmappedColumns = includeUnmappedColumns;
}
~ColumnMaskStream() =>
Disposable.dispose(ref sub);
protected override void myOnFirstSubscriber() {
Log.isnull(sub);
sub = source.Subscribe(OnSourceChanged);
void OnSourceChanged(IDataPacket dp) {
ensureOutputSchema(dp.Schema);
var addedRows = new ColumnMaskTable(dp.addedRows, columnMappings, outputSchema);
var deletedRows = new ColumnMaskTable(dp.deletedRows, columnMappings, outputSchema);
var allRows = new ColumnMaskTable(dp.newRows, columnMappings, outputSchema);
myFireEvent(new DataPacket(allRows, addedRows, deletedRows));
}
}
protected override void myOnLastSubscriber() =>
Disposable.dispose(ref sub);
[MemberNotNull(nameof(outputSchema), nameof(columnMappings))]
void ensureOutputSchema(IBlueTableSchema inputSchema) {
if (outputSchema is not null) {
Log.notnull(columnMappings);
return;
}
if (columnMaskExpression is not null) {
outputSchema = ColumnMaskTable.GetOutputSchema(inputSchema, columnMaskExpression, out columnMappings);
} else if (columnNameMappings is not null) {
outputSchema = ColumnMaskTable.GetOutputSchema(inputSchema, includeUnmappedColumns, columnNameMappings, out columnMappings);
} else {
throw new InvalidOperationException("Cannot build output schema");
}
}
}
@@ -0,0 +1,120 @@
namespace bts.BlueOrb;
using BCG;
using bts.collections;
using bts.utils;
using Bts.DataStreamCore;
using System;
using System.Collections.Generic;
using System.Linq;
public class ColumnMaskTable : IBlueRowCollection {
readonly IBlueRowCollection inputRows;
readonly (string? Name, int Index)[] columnMap;
public ColumnMaskTable(IBlueRowCollection inputRows, (string? Name, int Index)[] columnMap, IBlueTableSchema schema) {
this.inputRows = inputRows;
this.columnMap = columnMap;
this.schema = schema;
}
public IBlueTableSchema schema { get; }
public int Count => inputRows.Count;
public IBlueRow this[int id] =>
new ColumnMaskRow(inputRows[id], columnMap, schema);
public static ColumnMaskTable Create(IBlueRowCollection table, string columnMaskExpression) {
var outputSchema = GetOutputSchema(table.schema, columnMaskExpression, out var columnMappings);
return new ColumnMaskTable(table, columnMappings, outputSchema);
}
public static IBlueTableSchema GetOutputSchema(IBlueTableSchema inputSchema, string columnMaskExpression, out (string? Name, int Index)[] columnMappings) {
bool includeOthers;
if (columnMaskExpression.StartsWith('!')) {
columnMaskExpression = columnMaskExpression[1..].TrimStart();
includeOthers = false;
} else {
includeOthers = true;
}
var map = columnMaskExpression.Split(' ').Select(MapColumn).ToOrderedDictionary(tup => tup.Index, tup => tup.Name);
var inputCols = inputSchema.columnInfo;
if (includeOthers) {
var implicitCols = inputCols.Where(c => !map.ContainsKey(c.colNo)).ToList();
map.AddRange(implicitCols.Select(c => new KeyValuePair<int, string?>(c.colNo, null)));
} else if (!map.Any(c => inputCols[c.Key].colName == "id") && inputCols.TryGetFirst(c => c.colName == "id", out var idColInfo)) {
// Add 'id' col if missing
map.Insert(0, new(idColInfo.colNo, "id"));
}
var outputCols = new ColumnInfo[map.Count];
for (int i = 0; i < outputCols.Length; i++) {
var inputColInfo = inputCols[map.Keys[i]];
outputCols[i] = new(map.Values[i] ?? inputColInfo.colName, i, inputColInfo.colType);
}
columnMappings = map.WrapSelector(kv => (kv.Value, kv.Key), null).ToArray();
return new BlueTableSchema(outputCols, inputSchema.tableName + "_ColumnMask");
(string? Name, int Index) MapColumn(string columnStr) {
var parts = columnStr.Split(':');
return parts.Length switch {
1 => ((string?)null, inputSchema[parts[0]].colNo),
2 => (parts[1], inputSchema[parts[0]].colNo),
_ => throw new InvalidOperationException("Unable to get output schema due to invalid column expression"),
};
}
}
public static IBlueTableSchema GetOutputSchema(IBlueTableSchema inputSchema, bool includeUnmapped, (string From, string To)[] columnNameMappings, out (string? OutputName, int InputIndex)[] columnMappings) {
var inputCols = inputSchema.columnInfo;
ColumnInfo[]? outputCols;
if (includeUnmapped) {
columnMappings = Array.ConvertAll(inputCols, c => ((string?)c.colName, c.colNo));
outputCols = ArrayUtils.Clone(inputCols);
foreach (var (from, to) in columnNameMappings) {
var idx = inputCols.ResolveIndices(from);
var c = outputCols[idx];
Log.assert(c.colNo == idx);
outputCols[idx] = new(to, c.colNo, c.colType);
columnMappings[idx] = (to, idx);
}
} else {
outputCols = new ColumnInfo[columnNameMappings.Length];
columnMappings = Array.ConvertAll(columnNameMappings, m => ((string?)m.To, inputCols.ResolveIndices(m.From)));
Array.Copy(inputCols, outputCols, inputCols.Length);
for (int i = 0; i < columnNameMappings.Length; i++) {
var (from, to) = columnNameMappings[i];
var idx = inputCols.ResolveIndices(from);
outputCols[idx] = new(to, i, inputCols[columnMappings[i].InputIndex].colType);
}
}
return new BlueTableSchema(outputCols, inputSchema.tableName + "_ColumnMask");
}
public bool containsID(int id) =>
inputRows.containsID(id);
public IEnumerator<IBlueRow> GetEnumerator() {
foreach (var r in inputRows) {
yield return new ColumnMaskRow(r, columnMap, schema);
}
}
public IBlueRow GetRow(int index) =>
new ColumnMaskRow(inputRows.GetRow(index), columnMap, schema);
public int IndexOf(IBlueRow item) {
int id = item.Id();
if (inputRows.containsID(id)) {
var inputRow = inputRows[id];
var outputRow = new ColumnMaskRow(inputRow, columnMap, schema);
if (BlueRowEqualityComparer.Equals(item, outputRow)) {
return inputRows.IndexOf(inputRow);
}
}
return -1;
}
}
@@ -4,6 +4,7 @@ using BCG;
using bts;
using bts.api.core;
using bts.BlueOrb;
using Bts.DataStreamCore;
using Bts.PubSub;
@@ -152,6 +153,14 @@ public static class DataStreamExtensions {
IBlueRowCollection? ApplyWrapper(EquatableKeyArrayWrapper groupKey, IBlueRowCollection groupRows) =>
apply(groupKey.Values, groupRows);
}
public static ViewStream RenameColumns(this ViewStream viewStream, bool includeUnmapped, params (string From, string To)[] columnMap) =>
new ColumnMaskStream(viewStream, includeUnmapped, columnMap);
public static IObservable<IDataPacket> TimeSeries(this IObservable<IDataPacket> source, int bufferSize, string timeColumn) =>
source.AsViewStream().TimeSeries(bufferSize, timeColumn).AsObservable();
public static ViewStream TimeSeries(this ViewStream source, int bufferSize, string timeColumn) =>
new TimeSeriesStream(source, bufferSize, timeColumn);
public static IDisposable Subscribe(this ViewStream viewStream, Action<IDataPacket> onChange) {
viewStream.onChange += OnChange;
@@ -0,0 +1,340 @@
namespace bts.BlueOrb;
using BCG;
using bts.api;
using bts.co2;
using bts.utils;
using Bts.DataStreamCore;
using Bts.SharedStreams.StreamFactories;
using Spark.Core.Collections;
using System.Collections.Immutable;
public class BboPriceSummaryRow : IBlueRow {
public static readonly IBlueTableSchema Schema =
new BlueTableSchemaBuilder("BboPriceSummary") {
(MyCols.IdName, typeof(int)),
WellKnownColumns.Timestamp,
(MyCols.BidTimeName, typeof(DateTime)),
(MyCols.BidTimeTilTradeName, typeof(TimeSpan)),
WellKnownColumns.BidPrice,
WellKnownColumns.BidQuantity,
(MyCols.BidMaxQuantityName, typeof(double)),
(MyCols.BidTradeQuantityName, typeof(double)),
(MyCols.BidInfluxQuantityName, typeof(double)),
(MyCols.BidUnfilledQuantityName, typeof(double)),
(MyCols.AskTimeName, typeof(DateTime)),
(MyCols.AskTimeTilTradeName, typeof(TimeSpan)),
WellKnownColumns.AskPrice,
WellKnownColumns.AskQuantity,
(MyCols.AskMaxQuantityName, typeof(double)),
(MyCols.AskTradeQuantityName, typeof(double)),
(MyCols.AskInfluxQuantityName, typeof(double)),
(MyCols.AskUnfilledQuantityName, typeof(double)),
(MyCols.TradeQuantityName, typeof(double)),
(MyCols.AvgTradePriceName, typeof(double)),
}
.ToSchema();
readonly ImmutableDictionary<int, BookRowData> books;
readonly ImmutableSortedSet<KeyValuePair<(DateTime Timestamp, int Id), BookRowData>> bookTimeseries;
readonly ImmutableDictionary<int, TradeRowData> trades;
readonly ImmutableSortedSet<KeyValuePair<(DateTime Timestamp, int Id), TradeRowData>> tradeTimeseries;
public BboPriceSummaryRow(int id)
: this(
id,
ImmutableDictionary<int, BookRowData>.Empty,
ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<(DateTime Timestamp, int Id), BookRowData>(TupleComparer.Comparer<DateTime, int>())),
ImmutableDictionary<int, TradeRowData>.Empty,
ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<(DateTime Timestamp, int Id), TradeRowData>(TupleComparer.Comparer<DateTime, int>()))) { }
public BboPriceSummaryRow(
int id,
ImmutableDictionary<int, BookRowData> books,
ImmutableSortedSet<KeyValuePair<(DateTime Timestamp, int Id), BookRowData>> bookTimeseries,
ImmutableDictionary<int, TradeRowData> trades,
ImmutableSortedSet<KeyValuePair<(DateTime Timestamp, int Id), TradeRowData>> tradeTimeseries
) {
Id = id;
this.books = books;
this.bookTimeseries = bookTimeseries;
this.trades = trades;
this.tradeTimeseries = tradeTimeseries;
if (bookTimeseries.TryGetFirst(out var bk)) {
BidPrice = bk.Value.BidPrice;
BidQuantity = bk.Value.BidQuantity;
BidMaxQuantity = bookTimeseries.Select(kv => kv.Value.BidQuantity).Except(double.NaN).MaxOrDefault();
AskPrice = bk.Value.AskPrice;
AskQuantity = bk.Value.AskQuantity;
AskMaxQuantity = bookTimeseries.Select(kv => kv.Value.AskQuantity).Except(double.NaN).MaxOrDefault();
}
}
public object? this[int index] =>
index switch {
MyCols.IdIndex => Id,
MyCols.TimestampIndex => OpenTimestamp,
MyCols.BidTimeIndex => BidTime,
MyCols.BidTimeTilTradeIndex => BidTimeTilTrade,
MyCols.BidPriceIndex => BidPrice,
MyCols.BidQuantityIndex => BidQuantity,
MyCols.BidMaxQuantityIndex => BidMaxQuantity,
MyCols.BidTradeQuantityIndex => BidTradeQuantity,
MyCols.BidInfluxQuantityIndex => BidInfluxQuantity,
MyCols.BidUnfilledQuantityIndex => BidUnfilledQuantity,
MyCols.AskTimeIndex => AskTime,
MyCols.AskTimeTilTradeIndex => AskTimeTilTrade,
MyCols.AskPriceIndex => AskPrice,
MyCols.AskQuantityIndex => AskQuantity,
MyCols.AskMaxQuantityIndex => AskMaxQuantity,
MyCols.AskTradeQuantityIndex => AskTradeQuantity,
MyCols.AskInfluxQuantityIndex => AskInfluxQuantity,
MyCols.AskUnfilledQuantityIndex => AskUnfilledQuantity,
MyCols.TradeQuantityIndex => TradeQuantity,
MyCols.AvgTradePriceIndex => AvgTradePrice,
_ => throw new KeyNotFoundException(),
};
public object? this[string colName] =>
colName switch {
MyCols.IdName => Id,
MyCols.TimestampName => OpenTimestamp,
MyCols.BidTimeName => BidTime,
MyCols.BidTimeTilTradeName => BidTimeTilTrade,
MyCols.BidPriceName => BidPrice,
MyCols.BidQuantityName => BidQuantity,
MyCols.BidMaxQuantityName => BidMaxQuantity,
MyCols.BidTradeQuantityName => BidTradeQuantity,
MyCols.BidInfluxQuantityName => BidInfluxQuantity,
MyCols.BidUnfilledQuantityName => BidUnfilledQuantity,
MyCols.AskTimeName => AskTime,
MyCols.AskTimeTilTradeName => AskTimeTilTrade,
MyCols.AskPriceName => AskPrice,
MyCols.AskQuantityName => AskQuantity,
MyCols.AskMaxQuantityName => AskMaxQuantity,
MyCols.AskTradeQuantityName => AskTradeQuantity,
MyCols.AskInfluxQuantityName => AskInfluxQuantity,
MyCols.AskUnfilledQuantityName => AskUnfilledQuantity,
MyCols.TradeQuantityName => TradeQuantity,
MyCols.AvgTradePriceName => AvgTradePrice,
_ => this[Schema.ResolveIndices(colName)],
};
public int Id { get; }
public DateTime? BidTime => bidPriceChangeRows().FirstOrDefault()?.Timestamp;
public DateTime? BidLastTime => reverseBidPriceChangeRows().FirstOrDefault()?.Timestamp;
public double BidLastQuantity => reverseBidPriceChangeRows().FirstOrDefault()?.BidQuantity ?? double.NaN;
public TimeSpan? BidPriceDuration => BidLastTime - BidTime;
/// <summary>
/// When the first trade occurs at the bid price
/// </summary>
public DateTime? BidTradeTime =>
tradeTimeseries.Where(kv => kv.Value.AggressorSide == AggressorSide.Sell || kv.Value.Price <= BidPrice).TryGetFirst(out var kv)
? kv.Value.Timestamp
: null;
public TimeSpan BidTimeTilTrade =>
BidTradeTime.HasValue && BidTime.HasValue
? BidTradeTime.Value - BidTime.Value
: TimeSpan.Zero;
public double BidPrice { get; }
public double BidQuantity { get; }
public double BidMaxQuantity { get; }
public double BidTradeQuantity => trades.Values.Where(t => t.AggressorSide == AggressorSide.Sell || t.Price <= BidPrice).Sum(t => t.Quantity);
public double BidInfluxQuantity =>
bookTimeseries
.PairWithPrevious(false)
.Select(p => p.Current.Value.BidQuantity - p.Previous.Value.BidQuantity)
.Where(d => d > 0)
.Sum();
public double BidUnfilledQuantity => BidQuantity + BidInfluxQuantity - BidTradeQuantity;
public DateTime? LastTradeTime =>
tradeTimeseries.Reverse().TryGetFirst(out var kv)
? kv.Value.Timestamp
: null;
public DateTime? OpenTimestamp {
get {
var rv = BidTime;
if (!rv.HasValue || (AskTime.HasValue && rv.Value > AskTime.Value)) {
rv = AskTime;
}
return rv;
}
}
public DateTime? CloseTimestamp {
get {
var rv = BidLastTime;
if (!rv.HasValue || (AskLastTime.HasValue && rv.Value > AskLastTime.Value)) {
rv = AskLastTime;
}
return rv;
}
}
public DateTime? AskTime => askPriceChangeRows().FirstOrDefault()?.Timestamp;
public DateTime? AskLastTime => reverseAskPriceChangeRows().FirstOrDefault()?.Timestamp;
public double AskLastQuantity => reverseAskPriceChangeRows().FirstOrDefault()?.AskQuantity ?? double.NaN;
public TimeSpan? AskPriceDuration => AskLastTime - AskTime;
public DateTime? AskTradeTime =>
tradeTimeseries.Where(kv => kv.Value.Price >= AskPrice).TryGetFirst(out var kv)
? kv.Value.Timestamp
: null;
public TimeSpan AskTimeTilTrade =>
AskTradeTime.HasValue && AskTime.HasValue
? AskTradeTime.Value - AskTime.Value
: TimeSpan.Zero;
public double AskPrice { get; }
public double AskQuantity { get; }
public double AskMaxQuantity { get; }
public double AskTradeQuantity => trades.Values.Where(t => t.AggressorSide == AggressorSide.Buy || t.Price >= AskPrice).Sum(t => t.Quantity);
public double AskInfluxQuantity =>
bookTimeseries
.PairWithPrevious(false)
.Select(p => p.Current.Value.AskQuantity - p.Previous.Value.AskQuantity)
.Where(d => d > 0)
.Sum();
public double AskUnfilledQuantity => AskQuantity + AskInfluxQuantity - AskTradeQuantity;
public double TradeQuantity => trades.Values.Sum(t => t.Quantity);
public double AvgTradePrice {
get {
var tqty = TradeQuantity;
if (!tqty.is_real_positive()) {
return double.NaN;
}
return trades.Values.Sum(t => t.Price * t.Quantity) / tqty;
}
}
IBlueTableSchema IBlueRow.schema => Schema;
public BboPriceSummaryRow OnBook(IBlueRow row) =>
OnBook(new BookRowData(row));
public BboPriceSummaryRow OnBook(BookRowData rowData) {
var booksBd = books.ToBuilder();
var bookTimeseriesBd = bookTimeseries.ToBuilder();
if (booksBd.TryGetValue(rowData.Id, out var oldRowData)) {
bookTimeseriesBd.Remove(new((oldRowData.Timestamp, oldRowData.Id), oldRowData));
}
booksBd[rowData.Id] = rowData;
bookTimeseriesBd.Add(new((rowData.Timestamp, rowData.Id), rowData));
return new(Id, booksBd.ToImmutable(), bookTimeseriesBd.ToImmutable(), trades, tradeTimeseries);
}
public BboPriceSummaryRow OnTrade(IBlueRow row) =>
OnTrade(new TradeRowData(row));
public BboPriceSummaryRow OnTrade(TradeRowData rowData) {
var tradesBd = trades.ToBuilder();
var timeseriesBd = tradeTimeseries.ToBuilder();
if (tradesBd.TryGetValue(rowData.Id, out var oldRowData)) {
timeseriesBd.Remove(new((oldRowData.Timestamp, oldRowData.Id), oldRowData));
}
tradesBd[rowData.Id] = rowData;
timeseriesBd.Add(new((rowData.Timestamp, rowData.Id), rowData));
return new(Id, books, bookTimeseries, tradesBd.ToImmutable(), timeseriesBd.ToImmutable());
}
IEnumerable<BookRowData> bidPriceChangeRows() =>
priceChangeRows(Side.Buy);
IEnumerable<BookRowData> reverseBidPriceChangeRows() =>
reversePriceChangeRows(Side.Buy);
IEnumerable<BookRowData> askPriceChangeRows() =>
priceChangeRows(Side.Sell);
IEnumerable<BookRowData> reverseAskPriceChangeRows() =>
reversePriceChangeRows(Side.Sell);
IEnumerable<BookRowData> priceChangeRows(Side side) {
using var en = bookTimeseries.GetEnumerator();
if (!en.MoveNext()) {
yield break;
}
var last = en.Current.Value;
yield return en.Current.Value;
while (en.MoveNext()) {
var (lastPrice, price) = side switch {
Side.Buy => (last.BidPrice, en.Current.Value.BidPrice),
Side.Sell => (last.AskPrice, en.Current.Value.AskPrice),
_ => throw new NotImplementedException(),
};
if (price.Equals(lastPrice)) {
continue;
}
yield return en.Current.Value;
last = en.Current.Value;
}
}
IEnumerable<BookRowData> reversePriceChangeRows(Side side) {
using var en = bookTimeseries.Reverse().GetEnumerator();
if (!en.MoveNext()) {
yield break;
}
BookRowData last = en.Current.Value;
while (en.MoveNext()) {
var (lastPrice, price) = side switch {
Side.Buy => (last.BidPrice, en.Current.Value.BidPrice),
Side.Sell => (last.AskPrice, en.Current.Value.AskPrice),
_ => throw new NotImplementedException(),
};
if (!price.Equals(lastPrice)) {
yield return last;
}
last = en.Current.Value;
}
yield return last;
}
class MyCols {
public const string IdName = "id";
public const string TimestampName = WellKnownColumnNames.Timestamp;
public const string BidTimeName = "bid_time";
public const string BidTimeTilTradeName = "bid_time_til_trade";
public const string BidPriceName = WellKnownColumnNames.BidPrice;
public const string BidQuantityName = WellKnownColumnNames.BidQuantity;
public const string BidMaxQuantityName = "bid_max_size";
public const string BidTradeQuantityName = "bid_trade_size";
public const string BidInfluxQuantityName = "bid_size_influx";
public const string BidUnfilledQuantityName = "bid_unfilled_size";
public const string AskTimeName = "ask_time";
public const string AskTimeTilTradeName = "ask_time_til_trade";
public const string AskPriceName = WellKnownColumnNames.AskPrice;
public const string AskQuantityName = WellKnownColumnNames.AskQuantity;
public const string AskMaxQuantityName = "ask_max_size";
public const string AskTradeQuantityName = "ask_trade_size";
public const string AskInfluxQuantityName = "ask_influx_size";
public const string AskUnfilledQuantityName = "ask_unfilled_size";
public const string TradeQuantityName = "trade_size";
public const string AvgTradePriceName = "avg_trade_price";
public const int IdIndex = 0;
public const int TimestampIndex = 1;
public const int BidTimeIndex = 2;
public const int BidTimeTilTradeIndex = 3;
public const int BidPriceIndex = 4;
public const int BidQuantityIndex = 5;
public const int BidMaxQuantityIndex = 6;
public const int BidTradeQuantityIndex = 7;
public const int BidInfluxQuantityIndex = 8;
public const int BidUnfilledQuantityIndex = 9;
public const int AskTimeIndex = 10;
public const int AskTimeTilTradeIndex = 11;
public const int AskPriceIndex = 12;
public const int AskQuantityIndex = 13;
public const int AskMaxQuantityIndex = 14;
public const int AskTradeQuantityIndex = 15;
public const int AskInfluxQuantityIndex = 16;
public const int AskUnfilledQuantityIndex = 17;
public const int TradeQuantityIndex = 18;
public const int AvgTradePriceIndex = 19;
}
}
@@ -0,0 +1,169 @@
namespace bts.BlueOrb;
using BCG;
using bts.api;
using bts.collections;
using Bts.DataStreamCore;
using Bts.PubSub;
using Bts.SharedStreams.StreamFactories;
using Spark.Core;
using Spark.Core.Collections;
using Spark.Core.Disposables;
using System.Collections.Immutable;
using System.Diagnostics;
[DataStreamUserRespresentation(false)]
public class BboPriceSummaryStream : ViewStream, IDisposable {
readonly IObservable<IDataPacket> bboStream;
readonly IObservable<IDataPacket> tradeStream;
readonly IWorkScheduler workScheduler;
readonly TimeSpan publishInterval;
IBlueRowCollection? lastFired;
ImmutableSortedSet<KeyValuePair<(DateTime Timestamp, int Id), BookRowData>> bookTimeseries =
ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<(DateTime, int), BookRowData>(TupleComparer.Comparer<DateTime, int>()));
ImmutableSortedSet<KeyValuePair<(DateTime Timestamp, int Id), TradeRowData>> tradeTimeseries =
ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<(DateTime, int), TradeRowData>(TupleComparer.Comparer<DateTime, int>()));
int isDirty = 0;
IDisposable? sub;
public BboPriceSummaryStream(IObservable<IDataPacket> bboStream, IObservable<IDataPacket> tradeStream, IWorkScheduler workScheduler, TimeSpan publishInterval) {
this.bboStream = bboStream;
this.tradeStream = tradeStream;
this.workScheduler = workScheduler;
this.publishInterval = publishInterval;
}
~BboPriceSummaryStream() =>
Dispose();
public IEnumerable<BboPriceSummaryRow> GetRows() {
int idCounter = 1;
var rows = bookTimeseries.AggregateConsecutiveGroups(
() => new BboPriceSummaryRow(idCounter++),
(r, kv) => r.OnBook(kv.Value),
kv => (kv.Value.BidPrice, kv.Value.AskPrice))
.ToList();
var tradeList = tradeTimeseries.ToList();
int tradePos = 0;
foreach (var (r, next) in rows.PairWithPrevious()) {
if (r is null) {
continue;
}
var rOut = r;
if (rOut.BidTime.HasValue || rOut.AskTime.HasValue) {
var openTime = r.OpenTimestamp.assert_value();
var closeTime = next.OpenTimestamp.assert_value();
while (tradePos < tradeList.Count) {
var trade = tradeList[tradePos].Value;
if (trade.Timestamp < openTime) {
continue;
}
if (trade.Timestamp >= closeTime) {
break;
}
rOut = rOut.OnTrade(tradeList[tradePos].Value);
tradePos++;
}
}
yield return rOut;
}
}
public void Dispose() {
GC.SuppressFinalize(this);
Disposable.dispose(ref sub);
}
protected override void myOnFirstSubscriber() {
sub = Disposable.Create(
bboStream.Subscribe(OnBbo),
tradeStream.Subscribe(OnTrade),
workScheduler.Subscribe(Publish, publishInterval));
void OnBbo(IDataPacket dp) {
var bookTimeseriesBd = bookTimeseries.ToBuilder();
foreach (var r in dp.deletedRows) {
var rd = new BookRowData(r);
Log.assert(bookTimeseriesBd.Remove(new((rd.Timestamp, rd.Id), default!)));
}
foreach (var r in dp.addedRows) {
var rd = new BookRowData(r);
Log.assert(bookTimeseriesBd.Add(new((rd.Timestamp, rd.Id), rd)));
}
bookTimeseries = bookTimeseriesBd.ToImmutable();
Volatile.Write(ref isDirty, 1);
}
void OnTrade(IDataPacket dp) {
var tradeTimeseriesBd = tradeTimeseries.ToBuilder();
foreach (var r in dp.deletedRows) {
var rd = new TradeRowData(r);
Log.assert(tradeTimeseriesBd.Remove(new((rd.Timestamp, rd.Id), rd)));
}
foreach (var r in dp.addedRows) {
var rd = new TradeRowData(r);
Log.assert(tradeTimeseriesBd.Add(new((rd.Timestamp, rd.Id), rd)));
}
tradeTimeseries = tradeTimeseriesBd.ToImmutable();
Volatile.Write(ref isDirty, 1);
}
void Publish() {
if (Interlocked.CompareExchange(ref isDirty, 0, 1) == 0) {
return;
}
var sw = Stopwatch.StartNew();
lastFired ??= new BlueTable(BboPriceSummaryRow.Schema);
var rows = GetRows().ToList();
var table = new BlueTableListWrapper(BboPriceSummaryRow.Schema, rows.WrapSelector(r => (IBlueRow)r, null));
var removes = new BlueTableListWrapper(BboPriceSummaryRow.Schema, lastFired.Where(r => !HasRow(table, r)).ToList());
var adds = new BlueTableListWrapper(BboPriceSummaryRow.Schema, table.Where(r => !HasRow(lastFired, r)).ToList());
lastFired = table;
sw.Stop();
if (sw.Elapsed > TimeSpan.FromMilliseconds(20)) {
Log.DataFlow.warn($$"""[bbo_price_change_stream_slow_pub.json]{ "dur": "{{sw.ElapsedMilliseconds}}ms", "num_books": {{bookTimeseries.Count}} "num_trades": {{tradeTimeseries.Count}}, "ts": "{{DateTime.Now}}"}""");
}
myFireEvent(new DataPacket(table, adds, removes));
bool HasRow(IBlueRowCollection? rows, IBlueRow row) {
if (rows is null) {
return false;
}
var id = row.Id();
return rows.containsID(id) && BlueRowEqualityComparer.Equals(rows[id], row);
}
}
}
protected override void myOnLastSubscriber() =>
Disposable.dispose(ref sub);
class BookSideRowData {
public BookSideRowData(Side side, IBlueRow row) {
Side = side;
Row = row;
Id = row.Id();
Timestamp = row.GetDateTime(WellKnownColumns.Timestamp.Name).assert_value();
var (priceCol, qtyCol) = side switch {
Side.Buy => (WellKnownColumns.BidPrice.Name, WellKnownColumns.BidQuantity.Name),
Side.Sell => (WellKnownColumns.AskPrice.Name, WellKnownColumns.AskQuantity.Name),
_ => throw new ArgumentException("Invalid side", nameof(side)),
};
Price = row.GetFloat64(priceCol).assert_value();
Quantity = row.GetFloat64(qtyCol).assert_value();
}
public Side Side { get; }
public IBlueRow Row { get; }
public int Id { get; }
public DateTime Timestamp { get; }
public double Price { get; }
public double Quantity { get; }
}
}
@@ -0,0 +1,28 @@
namespace bts.BlueOrb;
using BCG;
using Bts.DataStreamCore;
using Bts.SharedStreams.StreamFactories;
using Spark.Core;
public class BookRowData {
public BookRowData(IBlueRow row) {
Row = row;
Id = row.Id();
Timestamp = row.GetDateTime(WellKnownColumns.Timestamp.Name).assert_value();
BidPrice = row.GetFloat64(WellKnownColumns.BidPrice.Name).assert_value();
BidQuantity = row.GetFloat64(WellKnownColumns.BidQuantity.Name).assert_value();
AskPrice = row.GetFloat64(WellKnownColumns.AskPrice.Name).assert_value();
AskQuantity = row.GetFloat64(WellKnownColumns.AskQuantity.Name).assert_value();
}
public IBlueRow Row { get; }
public int Id { get; }
public DateTime Timestamp { get; }
public double BidPrice { get; }
public double BidQuantity { get; }
public double AskPrice { get; }
public double AskQuantity { get; }
}
@@ -0,0 +1,28 @@
namespace bts.BlueOrb;
using BCG;
using bts.co2;
using Bts.DataStreamCore;
using Bts.SharedStreams.StreamFactories;
using Spark.Core;
public class TradeRowData {
public TradeRowData(IBlueRow row) {
Row = row;
Id = row.Id();
Timestamp = row.GetDateTime(WellKnownColumns.Timestamp.Name).assert_value();
AggressorSide = row.GetEnum<AggressorSide>("aggressor_side") ?? AggressorSide.None;
Price = row.GetFloat64(WellKnownColumns.Price.Name).assert_value();
Quantity = row.GetFloat64(WellKnownColumns.Quantity.Name).assert_value();
}
public IBlueRow Row { get; }
public int Id { get; }
public DateTime Timestamp { get; }
public AggressorSide AggressorSide { get; }
public double Price { get; }
public double Quantity { get; }
}
@@ -2,38 +2,90 @@ namespace Bts.SharedStreams.StreamFactories;
using BCG;
using bts.BlueOrb;
using bts.collections;
using Bts.DataStream;
using Bts.DataStreamCore;
using Bts.DataStreamCore.StreamFactories;
using Bts.PubSub;
using Microsoft.Extensions.Logging;
using Spark.Api.MarketData;
using Spark.Core.Reactive;
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Reactive.Linq;
public class BookStreamFactory : DataStreamFactory, IRecursiveDataStreamFactory {
readonly LiveBboStreamFactory liveBboFactory;
readonly IMarketDataClient? mdClient;
readonly ILogger logger;
readonly IWorkScheduler workScheduler;
IDataStreamFactory? compositeFactory = null;
public BookStreamFactory(IMarketDataClient? mdClient, ILogger<BookStreamFactory> logger) {
this.Add<BookStreamParams>("book", CreateBook, logger);
this.Add<BookStreamParams>(new[] { "bbo" }, CreateBbo, logger);
public BookStreamFactory(IMarketDataClient? mdClient, ILogger<BookStreamFactory> logger, ViewStreamDatabase workScheduler) {
this.mdClient = mdClient;
this.logger = logger;
this.workScheduler = workScheduler;
this.Add<BookStreamParams>(new[] { "book" }, CreateBook, logger, TryGetBookStreamParams);
this.Add<BookStreamParams>(new[] { "bbo" }, CreateBbo, logger, TryGetBookStreamParams);
this.Add<BookStreamParams>(new[] { "bboPriceSummary" }, createBboPriceSummary, logger, TryGetBookStreamParams);
liveBboFactory = new LiveBboStreamFactory(mdClient, logger);
Add(liveBboFactory);
ViewStream? CreateBook(BookStreamParams streamParams) =>
mdClient?.Books(streamParams.GetSecurityCriteria())
?.AsDataPackets(streamParams.GetSchemaBuilder().BookTupleAdapter())
?.AsViewStream()
?? throw new InvalidOperationException("Live md not available");
bool TryGetBookStreamParams(IEnumerable<(string? Name, int? Index, object? Value)> args, [NotNullWhen(true)]out BookStreamParams? result, [NotNullWhen(false)]out string? errorMsg) {
var argList = args.ToList();
if (argList.Count == 0) {
errorMsg = "Missing required parameters (symbol)";
result = null;
return false;
}
if (!StreamFactoryArgStore.TryGetParams(args, out result, out errorMsg)) {
result = new();
}
var mdArgStore = new MarketDataArgStore(argList, "bboPriceSummary", logger);
if (!result.Venue.HasValue) {
result.Venue = mdArgStore.GetVenue(required: false);
}
if (string.IsNullOrEmpty(result.Symbol)) {
result.Symbol = mdArgStore.GetSymbol(required: false);
}
if (string.IsNullOrEmpty(result.SecurityName)) {
result.SecurityName = mdArgStore.GetSecurityName(required: false);
}
if (!result.SecurityType.HasValue) {
result.SecurityType = mdArgStore.GetSecurityType(required: false);
}
if (string.IsNullOrEmpty(result.Symbol) && string.IsNullOrEmpty(result.SecurityName)) {
errorMsg = "Missing required parameters (symbol OR securityName)";
return false;
}
errorMsg = null;
return true;
}
ViewStream? CreateBook(BookStreamParams streamParams) {
if (!checkHasLiveMd()) {
return null;
}
return this.mdClient.Books(streamParams.GetSecurityCriteria())
.AsDataPackets(streamParams.GetSchemaBuilder().BookTupleAdapter())
.AsViewStream()
.TimeSeries(streamParams.BufferSize, WellKnownColumnNames.Timestamp);
}
ViewStream? CreateBbo(BookStreamParams streamParams) {
return mdClient?.Books(streamParams.GetSecurityCriteria())
?.AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter())
?.AsViewStream()
?? throw new InvalidOperationException("Live md not available");
if (!checkHasLiveMd()) {
return null;
}
return this.mdClient.Books(streamParams.GetSecurityCriteria())
.AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter())
.AsViewStream()
.TimeSeries(streamParams.BufferSize, WellKnownColumnNames.Timestamp);
}
}
@@ -42,6 +94,46 @@ public class BookStreamFactory : DataStreamFactory, IRecursiveDataStreamFactory
liveBboFactory.OnCompositeFactory(compositeFactory);
}
ViewStream? createBboPriceSummary(BookStreamParams streamParams) {
if (!checkHasLiveMd()) {
return null;
}
var secCriteria = streamParams.GetSecurityCriteria();
mdClient.Books(secCriteria).WithHistory().Select(t => t.Table);
// var bboStream = mdClient.Books(secCriteria).AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter());
var bboStream = liveBboFactory.Create(streamParams)
?? throw new ArgumentException("Failed to create bbo stream", nameof(streamParams));
var bboTimeSeriesStream = new TimeSeriesStream(bboStream, streamParams.BufferSize, WellKnownColumnNames.Timestamp);
ViewStream? tradeStream = null;
if (streamParams.Symbol == "ES" && compositeFactory is not null) {
tradeStream = compositeFactory.Create("ReplayFutureLiveTradeStream", new (string?, int?, object?)[] { ("contract", 0, streamParams.Symbol) });
if (tradeStream is null) {
logger.LogError("Failed to create trade stream for bboPriceSummary stream");
return null;
}
tradeStream = tradeStream.RenameColumns(true, ("trade_price", WellKnownColumnNames.Price), ("trade_size", WellKnownColumnNames.Quantity));
} else {
var tradeSchemaBd = new TradeSchemaBuilder();
var tradeSchema = tradeSchemaBd.ToSchema();
var tradeAdapter = tradeSchemaBd.TradeInfoAdapter();
tradeStream = TradeStreamFactory.TradeStream(mdClient.Trades(secCriteria), tradeSchema, tradeAdapter).AsViewStream();
}
var bboPriceSummaryStream = new BboPriceSummaryStream(bboStream.AsObservable(), tradeStream.AsObservable(), workScheduler, TimeSpan.FromSeconds(1));
return bboPriceSummaryStream;
}
static bool checkHasLiveMd([NotNullWhen(true)] IMarketDataClient? mdClient, ILogger? logger = null) {
if (mdClient is not null) {
return true;
}
logger?.LogError("Market data is unavailable (md client is null)");
return false;
}
[MemberNotNullWhen(true, nameof(mdClient))]
bool checkHasLiveMd() =>
checkHasLiveMd(mdClient, logger);
class LiveBboStreamFactory : IDataStreamFactory, IRecursiveDataStreamFactory {
static readonly IDataStreamMetadata metadata = new DataStreamMetadata() {
Name = "LiveBBOStream",
@@ -110,9 +202,26 @@ public class BookStreamFactory : DataStreamFactory, IRecursiveDataStreamFactory
}
}
public ViewStream? Create(BookStreamParams streamParams) {
var secType = streamParams.SecurityType;
if (streamParams.Symbol == "ES" && (!secType.HasValue || secType.Value == bts.api.SecurityType.Future) && compositeFactory is not null) {
return compositeFactory.Create("ReplayFutureLiveBBOStream", new (string?, int?, object?)[] { ("contract", 0, streamParams.Symbol) });
}
if (!checkHasLiveMd()) {
return null;
}
return mdClient.Books(streamParams.GetSecurityCriteria())
.AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter())
.AsViewStream();
}
public IEnumerable<IDataStreamMetadata> GetSupportedStreams() {
yield return metadata;
}
[MemberNotNullWhen(true, nameof(mdClient))]
bool checkHasLiveMd() =>
BookStreamFactory.checkHasLiveMd(mdClient, logger);
}
public class LiveBboStreamParams {
@@ -32,6 +32,9 @@ public class BookStreamParams {
[StreamArgument(RegexNames = new[] { "$(show|include)?(cashPerPoint|cpp)^" })]
public bool ShowCashPerPoint { get; set; } = true;
[StreamArgument(RegexNames = new[] { "$(buffer|size|bufferSize|cap|capacity|len|length)^" })]
public int BufferSize { get; set; } = 10_000;
public mbts.spark.SecurityCriteria GetSecurityCriteria() {
var clause = new mbts.spark.SecurityCriteriaClause(
exchanges: Venue.HasValue ? new[] { Venue.Value } : null,
@@ -327,6 +327,21 @@ namespace System.Collections.Generic {
}
}
public static IEnumerable<(T? Previous, T Current)> PairWithPrevious<T>(this IEnumerable<T> source, bool includeZeroIndex = true) {
using var en = source.GetEnumerator();
if (!en.MoveNext()) {
yield break;
}
var prev = en.Current;
if (includeZeroIndex) {
yield return (default, prev);
}
while (en.MoveNext()) {
yield return (prev, en.Current);
prev = en.Current;
}
}
/// <summary>
/// Checks if <paramref name="source"/> implements <see cref="ICollection{T}"/> or <see cref="IReadOnlyCollection{T}"/>. If so, the resultant Count is returned, otherwise null (will not enumerate through any items).
/// </summary>
@@ -486,6 +501,19 @@ namespace System.Collections.Generic {
return false;
}
public static bool TryGetLast<T>(this IEnumerable<T> source, [MaybeNullWhen(false)]out T result) {
ArgumentNullException.ThrowIfNull(source);
using var en = source.GetEnumerator();
if (!en.MoveNext()) {
result = default;
return false;
}
do {
result = en.Current;
} while (en.MoveNext());
return true;
}
public static bool TryGetSingle<T>(this IEnumerable<T> source, out T? result) {
if (source == null) throw new ArgumentNullException(nameof(source));
using (var en = source.GetEnumerator()) {
@@ -0,0 +1,97 @@
#nullable enable
namespace Spark.Core.Reactive;
using bts;
using bts.api.core;
using Spark.Core.Collections;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Reactive.Linq;
public static class TableUpdateWithHistory {
public const int DEFAULT_CAPACITY = 16;
public static TableUpdateWithHistory<TKey, TValue> Empty<TKey, TValue>(int capacity = DEFAULT_CAPACITY) where TKey : IEquatable<TKey> =>
new();
public static IObservable<TableUpdateWithHistory<TKey, TValue>> WithHistory<TKey, TValue>(this IObservable<TableUpdate<TValue, TKey>> source) where TKey : IEquatable<TKey> {
return source.Scan(Empty<TKey, TValue>(), AddHistory);
TableUpdateWithHistory<TKey, TValue> AddHistory(TableUpdateWithHistory<TKey, TValue> history, TableUpdate<TValue, TKey> update) =>
history.Update(update);
}
}
public class TableUpdateWithHistory<TKey, TValue> where TKey : IEquatable<TKey> {
readonly CircularBuffer<TableUpdate<TValue, TKey>> buffer;
readonly int capacity;
readonly bool dropOnRemove;
readonly TableUpdate<Record, TKey> table = TableUpdate<Record, TKey>.mk_empty(r => r.Key);
readonly long version;
public TableUpdateWithHistory(int capacity = TableUpdateWithHistory.DEFAULT_CAPACITY, bool dropOnRemove = true)
: this(TableUpdate<Record, TKey>.mk_empty(r => r.Key), 0, capacity, dropOnRemove) { }
TableUpdateWithHistory(TableUpdate<Record, TKey> table, long version, int capacity, bool dropOnRemove = true) {
buffer = new(capacity);
this.capacity = capacity;
this.version = version;
this.dropOnRemove = dropOnRemove;
}
public TableUpdate<Record, TKey> Table => table;
public TableUpdateWithHistory<TKey, TValue> Update(TableUpdate<TValue, TKey> upd) {
var getKey = Log.notnull(upd.key_extractor);
long version = this.version + 1;
buffer.Add(upd);
var bd = table.to_builder();
var time = DateTime.Now;
if (dropOnRemove) {
foreach (var v in upd.removed) {
bd.remove(getKey(v));
}
}
foreach (var v in upd.changed.Concat(upd.added)) {
var k = getKey(v);
if (!bd.TryGetValue(k, out var rec)) {
rec = new(k);
}
rec = rec.Update(v, time, version, capacity);
bd.add_or_update(rec);
}
return new(bd.to_table(), version, capacity, dropOnRemove);
}
public readonly struct Record {
public readonly ImmutableList<(TValue Value, DateTime Time, long UpdateNum)>? Buffer;
public Record(TKey key, ImmutableList<(TValue Value, DateTime Time, long UpdateNum)>? buffer = null) {
Key = key;
Buffer = buffer;
}
public TKey Key { get; }
[MemberNotNullWhen(false, nameof(Buffer))]
public bool IsEmpty => Count == 0;
public int Count => Buffer?.Count ?? 0;
public TValue Value => Buffer?.TryGetLast(out var lst) ?? false
? lst.Value
: throw new InvalidOperationException("Buffer is empty");
public Record Update(TValue value, DateTime time, long updateNum, int capacity) {
var buffer = Buffer ?? ImmutableList<(TValue Value, DateTime Time, long UpdateNum)>.Empty.Add((value, time, updateNum));
if (buffer.Count >= capacity) {
buffer = buffer.RemoveRange(0, buffer.Count - capacity + 1);
}
return new(Key, buffer);
}
}
}