diff --git a/src/DataStreamSuite/DataStream/BlueOrb/ColumnMaskRow.cs b/src/DataStreamSuite/DataStream/BlueOrb/ColumnMaskRow.cs new file mode 100644 index 000000000..fae5fadf2 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/ColumnMaskRow.cs @@ -0,0 +1,19 @@ +namespace bts.BlueOrb; + +using BCG; + +public class ColumnMaskRow : IBlueRow { + readonly IBlueRow inputRow; + readonly (string? Name, int Index)[] columnMap; + + public ColumnMaskRow(IBlueRow inputRow, (string? Name, int Index)[] columnMap, IBlueTableSchema schema) { + this.inputRow = inputRow; + this.columnMap = columnMap; + this.schema = schema; + } + + public IBlueTableSchema schema { get; } + + public object? this[int index] => + inputRow[columnMap[index].Index]; +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/ColumnMaskStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/ColumnMaskStream.cs new file mode 100644 index 000000000..b53f82436 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/ColumnMaskStream.cs @@ -0,0 +1,68 @@ +namespace bts.BlueOrb; + +using BCG; + +using Bts.DataStream; + +using Spark.Core.Disposables; + +using System; +using System.Diagnostics.CodeAnalysis; + +public class ColumnMaskStream : ViewStream { + readonly ViewStream source; + readonly string? columnMaskExpression; + + IDisposable? sub; + IBlueTableSchema? outputSchema; + // Each item in this array represents an output column as a tuple of + // new name, and corresponding index in the input schema. + (string? OutputName, int InputIndex)[]? columnMappings; + (string From, string To)[]? columnNameMappings; + readonly bool includeUnmappedColumns; + + public ColumnMaskStream(ViewStream source, string columnMaskExpression) { + this.source = source; + this.columnMaskExpression = columnMaskExpression; + } + + public ColumnMaskStream(ViewStream source, bool includeUnmappedColumns, params (string From, string To)[] columnNameMappings) { + this.source = source; + this.columnNameMappings = columnNameMappings; + this.includeUnmappedColumns = includeUnmappedColumns; + } + + ~ColumnMaskStream() => + Disposable.dispose(ref sub); + + protected override void myOnFirstSubscriber() { + Log.isnull(sub); + sub = source.Subscribe(OnSourceChanged); + + void OnSourceChanged(IDataPacket dp) { + ensureOutputSchema(dp.Schema); + var addedRows = new ColumnMaskTable(dp.addedRows, columnMappings, outputSchema); + var deletedRows = new ColumnMaskTable(dp.deletedRows, columnMappings, outputSchema); + var allRows = new ColumnMaskTable(dp.newRows, columnMappings, outputSchema); + myFireEvent(new DataPacket(allRows, addedRows, deletedRows)); + } + } + + protected override void myOnLastSubscriber() => + Disposable.dispose(ref sub); + + [MemberNotNull(nameof(outputSchema), nameof(columnMappings))] + void ensureOutputSchema(IBlueTableSchema inputSchema) { + if (outputSchema is not null) { + Log.notnull(columnMappings); + return; + } + if (columnMaskExpression is not null) { + outputSchema = ColumnMaskTable.GetOutputSchema(inputSchema, columnMaskExpression, out columnMappings); + } else if (columnNameMappings is not null) { + outputSchema = ColumnMaskTable.GetOutputSchema(inputSchema, includeUnmappedColumns, columnNameMappings, out columnMappings); + } else { + throw new InvalidOperationException("Cannot build output schema"); + } + } +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/ColumnMaskTable.cs b/src/DataStreamSuite/DataStream/BlueOrb/ColumnMaskTable.cs new file mode 100644 index 000000000..7aa4a098e --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/ColumnMaskTable.cs @@ -0,0 +1,120 @@ +namespace bts.BlueOrb; + +using BCG; + +using bts.collections; +using bts.utils; + +using Bts.DataStreamCore; + +using System; +using System.Collections.Generic; +using System.Linq; + +public class ColumnMaskTable : IBlueRowCollection { + readonly IBlueRowCollection inputRows; + readonly (string? Name, int Index)[] columnMap; + + public ColumnMaskTable(IBlueRowCollection inputRows, (string? Name, int Index)[] columnMap, IBlueTableSchema schema) { + this.inputRows = inputRows; + this.columnMap = columnMap; + this.schema = schema; + } + + public IBlueTableSchema schema { get; } + + public int Count => inputRows.Count; + + public IBlueRow this[int id] => + new ColumnMaskRow(inputRows[id], columnMap, schema); + + public static ColumnMaskTable Create(IBlueRowCollection table, string columnMaskExpression) { + var outputSchema = GetOutputSchema(table.schema, columnMaskExpression, out var columnMappings); + return new ColumnMaskTable(table, columnMappings, outputSchema); + } + + public static IBlueTableSchema GetOutputSchema(IBlueTableSchema inputSchema, string columnMaskExpression, out (string? Name, int Index)[] columnMappings) { + bool includeOthers; + if (columnMaskExpression.StartsWith('!')) { + columnMaskExpression = columnMaskExpression[1..].TrimStart(); + includeOthers = false; + } else { + includeOthers = true; + } + var map = columnMaskExpression.Split(' ').Select(MapColumn).ToOrderedDictionary(tup => tup.Index, tup => tup.Name); + var inputCols = inputSchema.columnInfo; + if (includeOthers) { + var implicitCols = inputCols.Where(c => !map.ContainsKey(c.colNo)).ToList(); + map.AddRange(implicitCols.Select(c => new KeyValuePair(c.colNo, null))); + } else if (!map.Any(c => inputCols[c.Key].colName == "id") && inputCols.TryGetFirst(c => c.colName == "id", out var idColInfo)) { + // Add 'id' col if missing + map.Insert(0, new(idColInfo.colNo, "id")); + } + var outputCols = new ColumnInfo[map.Count]; + for (int i = 0; i < outputCols.Length; i++) { + var inputColInfo = inputCols[map.Keys[i]]; + outputCols[i] = new(map.Values[i] ?? inputColInfo.colName, i, inputColInfo.colType); + } + columnMappings = map.WrapSelector(kv => (kv.Value, kv.Key), null).ToArray(); + return new BlueTableSchema(outputCols, inputSchema.tableName + "_ColumnMask"); + + (string? Name, int Index) MapColumn(string columnStr) { + var parts = columnStr.Split(':'); + return parts.Length switch { + 1 => ((string?)null, inputSchema[parts[0]].colNo), + 2 => (parts[1], inputSchema[parts[0]].colNo), + _ => throw new InvalidOperationException("Unable to get output schema due to invalid column expression"), + }; + } + } + + public static IBlueTableSchema GetOutputSchema(IBlueTableSchema inputSchema, bool includeUnmapped, (string From, string To)[] columnNameMappings, out (string? OutputName, int InputIndex)[] columnMappings) { + var inputCols = inputSchema.columnInfo; + ColumnInfo[]? outputCols; + if (includeUnmapped) { + columnMappings = Array.ConvertAll(inputCols, c => ((string?)c.colName, c.colNo)); + outputCols = ArrayUtils.Clone(inputCols); + foreach (var (from, to) in columnNameMappings) { + var idx = inputCols.ResolveIndices(from); + var c = outputCols[idx]; + Log.assert(c.colNo == idx); + outputCols[idx] = new(to, c.colNo, c.colType); + columnMappings[idx] = (to, idx); + } + } else { + outputCols = new ColumnInfo[columnNameMappings.Length]; + columnMappings = Array.ConvertAll(columnNameMappings, m => ((string?)m.To, inputCols.ResolveIndices(m.From))); + Array.Copy(inputCols, outputCols, inputCols.Length); + for (int i = 0; i < columnNameMappings.Length; i++) { + var (from, to) = columnNameMappings[i]; + var idx = inputCols.ResolveIndices(from); + outputCols[idx] = new(to, i, inputCols[columnMappings[i].InputIndex].colType); + } + } + return new BlueTableSchema(outputCols, inputSchema.tableName + "_ColumnMask"); + } + + public bool containsID(int id) => + inputRows.containsID(id); + + public IEnumerator GetEnumerator() { + foreach (var r in inputRows) { + yield return new ColumnMaskRow(r, columnMap, schema); + } + } + + public IBlueRow GetRow(int index) => + new ColumnMaskRow(inputRows.GetRow(index), columnMap, schema); + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + if (inputRows.containsID(id)) { + var inputRow = inputRows[id]; + var outputRow = new ColumnMaskRow(inputRow, columnMap, schema); + if (BlueRowEqualityComparer.Equals(item, outputRow)) { + return inputRows.IndexOf(inputRow); + } + } + return -1; + } +} diff --git a/src/DataStreamSuite/DataStream/DataStreamExtensions.cs b/src/DataStreamSuite/DataStream/DataStreamExtensions.cs index e6100b048..5f7f6626d 100644 --- a/src/DataStreamSuite/DataStream/DataStreamExtensions.cs +++ b/src/DataStreamSuite/DataStream/DataStreamExtensions.cs @@ -4,6 +4,7 @@ using BCG; using bts; using bts.api.core; +using bts.BlueOrb; using Bts.DataStreamCore; using Bts.PubSub; @@ -152,6 +153,14 @@ public static class DataStreamExtensions { IBlueRowCollection? ApplyWrapper(EquatableKeyArrayWrapper groupKey, IBlueRowCollection groupRows) => apply(groupKey.Values, groupRows); } + public static ViewStream RenameColumns(this ViewStream viewStream, bool includeUnmapped, params (string From, string To)[] columnMap) => + new ColumnMaskStream(viewStream, includeUnmapped, columnMap); + + public static IObservable TimeSeries(this IObservable source, int bufferSize, string timeColumn) => + source.AsViewStream().TimeSeries(bufferSize, timeColumn).AsObservable(); + + public static ViewStream TimeSeries(this ViewStream source, int bufferSize, string timeColumn) => + new TimeSeriesStream(source, bufferSize, timeColumn); public static IDisposable Subscribe(this ViewStream viewStream, Action onChange) { viewStream.onChange += OnChange; diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BrandonChen/BboPriceSummaryRow.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BrandonChen/BboPriceSummaryRow.cs new file mode 100644 index 000000000..cfccb3f65 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BrandonChen/BboPriceSummaryRow.cs @@ -0,0 +1,340 @@ +namespace bts.BlueOrb; + +using BCG; + +using bts.api; +using bts.co2; +using bts.utils; + +using Bts.DataStreamCore; +using Bts.SharedStreams.StreamFactories; + +using Spark.Core.Collections; + +using System.Collections.Immutable; + +public class BboPriceSummaryRow : IBlueRow { + public static readonly IBlueTableSchema Schema = + new BlueTableSchemaBuilder("BboPriceSummary") { + (MyCols.IdName, typeof(int)), + WellKnownColumns.Timestamp, + (MyCols.BidTimeName, typeof(DateTime)), + (MyCols.BidTimeTilTradeName, typeof(TimeSpan)), + WellKnownColumns.BidPrice, + WellKnownColumns.BidQuantity, + (MyCols.BidMaxQuantityName, typeof(double)), + (MyCols.BidTradeQuantityName, typeof(double)), + (MyCols.BidInfluxQuantityName, typeof(double)), + (MyCols.BidUnfilledQuantityName, typeof(double)), + (MyCols.AskTimeName, typeof(DateTime)), + (MyCols.AskTimeTilTradeName, typeof(TimeSpan)), + WellKnownColumns.AskPrice, + WellKnownColumns.AskQuantity, + (MyCols.AskMaxQuantityName, typeof(double)), + (MyCols.AskTradeQuantityName, typeof(double)), + (MyCols.AskInfluxQuantityName, typeof(double)), + (MyCols.AskUnfilledQuantityName, typeof(double)), + (MyCols.TradeQuantityName, typeof(double)), + (MyCols.AvgTradePriceName, typeof(double)), + } + .ToSchema(); + + readonly ImmutableDictionary books; + readonly ImmutableSortedSet> bookTimeseries; + readonly ImmutableDictionary trades; + readonly ImmutableSortedSet> tradeTimeseries; + + public BboPriceSummaryRow(int id) + : this( + id, + ImmutableDictionary.Empty, + ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<(DateTime Timestamp, int Id), BookRowData>(TupleComparer.Comparer())), + ImmutableDictionary.Empty, + ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<(DateTime Timestamp, int Id), TradeRowData>(TupleComparer.Comparer()))) { } + + public BboPriceSummaryRow( + int id, + ImmutableDictionary books, + ImmutableSortedSet> bookTimeseries, + ImmutableDictionary trades, + ImmutableSortedSet> tradeTimeseries + ) { + Id = id; + this.books = books; + this.bookTimeseries = bookTimeseries; + this.trades = trades; + this.tradeTimeseries = tradeTimeseries; + if (bookTimeseries.TryGetFirst(out var bk)) { + BidPrice = bk.Value.BidPrice; + BidQuantity = bk.Value.BidQuantity; + BidMaxQuantity = bookTimeseries.Select(kv => kv.Value.BidQuantity).Except(double.NaN).MaxOrDefault(); + AskPrice = bk.Value.AskPrice; + AskQuantity = bk.Value.AskQuantity; + AskMaxQuantity = bookTimeseries.Select(kv => kv.Value.AskQuantity).Except(double.NaN).MaxOrDefault(); + } + } + + public object? this[int index] => + index switch { + MyCols.IdIndex => Id, + MyCols.TimestampIndex => OpenTimestamp, + MyCols.BidTimeIndex => BidTime, + MyCols.BidTimeTilTradeIndex => BidTimeTilTrade, + MyCols.BidPriceIndex => BidPrice, + MyCols.BidQuantityIndex => BidQuantity, + MyCols.BidMaxQuantityIndex => BidMaxQuantity, + MyCols.BidTradeQuantityIndex => BidTradeQuantity, + MyCols.BidInfluxQuantityIndex => BidInfluxQuantity, + MyCols.BidUnfilledQuantityIndex => BidUnfilledQuantity, + MyCols.AskTimeIndex => AskTime, + MyCols.AskTimeTilTradeIndex => AskTimeTilTrade, + MyCols.AskPriceIndex => AskPrice, + MyCols.AskQuantityIndex => AskQuantity, + MyCols.AskMaxQuantityIndex => AskMaxQuantity, + MyCols.AskTradeQuantityIndex => AskTradeQuantity, + MyCols.AskInfluxQuantityIndex => AskInfluxQuantity, + MyCols.AskUnfilledQuantityIndex => AskUnfilledQuantity, + MyCols.TradeQuantityIndex => TradeQuantity, + MyCols.AvgTradePriceIndex => AvgTradePrice, + _ => throw new KeyNotFoundException(), + }; + + public object? this[string colName] => + colName switch { + MyCols.IdName => Id, + MyCols.TimestampName => OpenTimestamp, + MyCols.BidTimeName => BidTime, + MyCols.BidTimeTilTradeName => BidTimeTilTrade, + MyCols.BidPriceName => BidPrice, + MyCols.BidQuantityName => BidQuantity, + MyCols.BidMaxQuantityName => BidMaxQuantity, + MyCols.BidTradeQuantityName => BidTradeQuantity, + MyCols.BidInfluxQuantityName => BidInfluxQuantity, + MyCols.BidUnfilledQuantityName => BidUnfilledQuantity, + MyCols.AskTimeName => AskTime, + MyCols.AskTimeTilTradeName => AskTimeTilTrade, + MyCols.AskPriceName => AskPrice, + MyCols.AskQuantityName => AskQuantity, + MyCols.AskMaxQuantityName => AskMaxQuantity, + MyCols.AskTradeQuantityName => AskTradeQuantity, + MyCols.AskInfluxQuantityName => AskInfluxQuantity, + MyCols.AskUnfilledQuantityName => AskUnfilledQuantity, + MyCols.TradeQuantityName => TradeQuantity, + MyCols.AvgTradePriceName => AvgTradePrice, + _ => this[Schema.ResolveIndices(colName)], + }; + + public int Id { get; } + + public DateTime? BidTime => bidPriceChangeRows().FirstOrDefault()?.Timestamp; + public DateTime? BidLastTime => reverseBidPriceChangeRows().FirstOrDefault()?.Timestamp; + public double BidLastQuantity => reverseBidPriceChangeRows().FirstOrDefault()?.BidQuantity ?? double.NaN; + public TimeSpan? BidPriceDuration => BidLastTime - BidTime; + /// + /// When the first trade occurs at the bid price + /// + public DateTime? BidTradeTime => + tradeTimeseries.Where(kv => kv.Value.AggressorSide == AggressorSide.Sell || kv.Value.Price <= BidPrice).TryGetFirst(out var kv) + ? kv.Value.Timestamp + : null; + public TimeSpan BidTimeTilTrade => + BidTradeTime.HasValue && BidTime.HasValue + ? BidTradeTime.Value - BidTime.Value + : TimeSpan.Zero; + public double BidPrice { get; } + public double BidQuantity { get; } + public double BidMaxQuantity { get; } + public double BidTradeQuantity => trades.Values.Where(t => t.AggressorSide == AggressorSide.Sell || t.Price <= BidPrice).Sum(t => t.Quantity); + public double BidInfluxQuantity => + bookTimeseries + .PairWithPrevious(false) + .Select(p => p.Current.Value.BidQuantity - p.Previous.Value.BidQuantity) + .Where(d => d > 0) + .Sum(); + public double BidUnfilledQuantity => BidQuantity + BidInfluxQuantity - BidTradeQuantity; + + public DateTime? LastTradeTime => + tradeTimeseries.Reverse().TryGetFirst(out var kv) + ? kv.Value.Timestamp + : null; + + public DateTime? OpenTimestamp { + get { + var rv = BidTime; + if (!rv.HasValue || (AskTime.HasValue && rv.Value > AskTime.Value)) { + rv = AskTime; + } + return rv; + } + } + public DateTime? CloseTimestamp { + get { + var rv = BidLastTime; + if (!rv.HasValue || (AskLastTime.HasValue && rv.Value > AskLastTime.Value)) { + rv = AskLastTime; + } + return rv; + } + } + public DateTime? AskTime => askPriceChangeRows().FirstOrDefault()?.Timestamp; + public DateTime? AskLastTime => reverseAskPriceChangeRows().FirstOrDefault()?.Timestamp; + public double AskLastQuantity => reverseAskPriceChangeRows().FirstOrDefault()?.AskQuantity ?? double.NaN; + public TimeSpan? AskPriceDuration => AskLastTime - AskTime; + public DateTime? AskTradeTime => + tradeTimeseries.Where(kv => kv.Value.Price >= AskPrice).TryGetFirst(out var kv) + ? kv.Value.Timestamp + : null; + public TimeSpan AskTimeTilTrade => + AskTradeTime.HasValue && AskTime.HasValue + ? AskTradeTime.Value - AskTime.Value + : TimeSpan.Zero; + public double AskPrice { get; } + public double AskQuantity { get; } + public double AskMaxQuantity { get; } + public double AskTradeQuantity => trades.Values.Where(t => t.AggressorSide == AggressorSide.Buy || t.Price >= AskPrice).Sum(t => t.Quantity); + public double AskInfluxQuantity => + bookTimeseries + .PairWithPrevious(false) + .Select(p => p.Current.Value.AskQuantity - p.Previous.Value.AskQuantity) + .Where(d => d > 0) + .Sum(); + public double AskUnfilledQuantity => AskQuantity + AskInfluxQuantity - AskTradeQuantity; + + public double TradeQuantity => trades.Values.Sum(t => t.Quantity); + public double AvgTradePrice { + get { + var tqty = TradeQuantity; + if (!tqty.is_real_positive()) { + return double.NaN; + } + return trades.Values.Sum(t => t.Price * t.Quantity) / tqty; + } + } + + IBlueTableSchema IBlueRow.schema => Schema; + + public BboPriceSummaryRow OnBook(IBlueRow row) => + OnBook(new BookRowData(row)); + + public BboPriceSummaryRow OnBook(BookRowData rowData) { + var booksBd = books.ToBuilder(); + var bookTimeseriesBd = bookTimeseries.ToBuilder(); + if (booksBd.TryGetValue(rowData.Id, out var oldRowData)) { + bookTimeseriesBd.Remove(new((oldRowData.Timestamp, oldRowData.Id), oldRowData)); + } + booksBd[rowData.Id] = rowData; + bookTimeseriesBd.Add(new((rowData.Timestamp, rowData.Id), rowData)); + return new(Id, booksBd.ToImmutable(), bookTimeseriesBd.ToImmutable(), trades, tradeTimeseries); + } + + public BboPriceSummaryRow OnTrade(IBlueRow row) => + OnTrade(new TradeRowData(row)); + + public BboPriceSummaryRow OnTrade(TradeRowData rowData) { + var tradesBd = trades.ToBuilder(); + var timeseriesBd = tradeTimeseries.ToBuilder(); + if (tradesBd.TryGetValue(rowData.Id, out var oldRowData)) { + timeseriesBd.Remove(new((oldRowData.Timestamp, oldRowData.Id), oldRowData)); + } + tradesBd[rowData.Id] = rowData; + timeseriesBd.Add(new((rowData.Timestamp, rowData.Id), rowData)); + return new(Id, books, bookTimeseries, tradesBd.ToImmutable(), timeseriesBd.ToImmutable()); + } + + IEnumerable bidPriceChangeRows() => + priceChangeRows(Side.Buy); + + IEnumerable reverseBidPriceChangeRows() => + reversePriceChangeRows(Side.Buy); + + IEnumerable askPriceChangeRows() => + priceChangeRows(Side.Sell); + + IEnumerable reverseAskPriceChangeRows() => + reversePriceChangeRows(Side.Sell); + + IEnumerable priceChangeRows(Side side) { + using var en = bookTimeseries.GetEnumerator(); + if (!en.MoveNext()) { + yield break; + } + var last = en.Current.Value; + yield return en.Current.Value; + while (en.MoveNext()) { + var (lastPrice, price) = side switch { + Side.Buy => (last.BidPrice, en.Current.Value.BidPrice), + Side.Sell => (last.AskPrice, en.Current.Value.AskPrice), + _ => throw new NotImplementedException(), + }; + if (price.Equals(lastPrice)) { + continue; + } + yield return en.Current.Value; + last = en.Current.Value; + } + } + + IEnumerable reversePriceChangeRows(Side side) { + using var en = bookTimeseries.Reverse().GetEnumerator(); + if (!en.MoveNext()) { + yield break; + } + BookRowData last = en.Current.Value; + while (en.MoveNext()) { + var (lastPrice, price) = side switch { + Side.Buy => (last.BidPrice, en.Current.Value.BidPrice), + Side.Sell => (last.AskPrice, en.Current.Value.AskPrice), + _ => throw new NotImplementedException(), + }; + if (!price.Equals(lastPrice)) { + yield return last; + } + last = en.Current.Value; + } + yield return last; + } + + class MyCols { + public const string IdName = "id"; + public const string TimestampName = WellKnownColumnNames.Timestamp; + public const string BidTimeName = "bid_time"; + public const string BidTimeTilTradeName = "bid_time_til_trade"; + public const string BidPriceName = WellKnownColumnNames.BidPrice; + public const string BidQuantityName = WellKnownColumnNames.BidQuantity; + public const string BidMaxQuantityName = "bid_max_size"; + public const string BidTradeQuantityName = "bid_trade_size"; + public const string BidInfluxQuantityName = "bid_size_influx"; + public const string BidUnfilledQuantityName = "bid_unfilled_size"; + public const string AskTimeName = "ask_time"; + public const string AskTimeTilTradeName = "ask_time_til_trade"; + public const string AskPriceName = WellKnownColumnNames.AskPrice; + public const string AskQuantityName = WellKnownColumnNames.AskQuantity; + public const string AskMaxQuantityName = "ask_max_size"; + public const string AskTradeQuantityName = "ask_trade_size"; + public const string AskInfluxQuantityName = "ask_influx_size"; + public const string AskUnfilledQuantityName = "ask_unfilled_size"; + public const string TradeQuantityName = "trade_size"; + public const string AvgTradePriceName = "avg_trade_price"; + + public const int IdIndex = 0; + public const int TimestampIndex = 1; + public const int BidTimeIndex = 2; + public const int BidTimeTilTradeIndex = 3; + public const int BidPriceIndex = 4; + public const int BidQuantityIndex = 5; + public const int BidMaxQuantityIndex = 6; + public const int BidTradeQuantityIndex = 7; + public const int BidInfluxQuantityIndex = 8; + public const int BidUnfilledQuantityIndex = 9; + public const int AskTimeIndex = 10; + public const int AskTimeTilTradeIndex = 11; + public const int AskPriceIndex = 12; + public const int AskQuantityIndex = 13; + public const int AskMaxQuantityIndex = 14; + public const int AskTradeQuantityIndex = 15; + public const int AskInfluxQuantityIndex = 16; + public const int AskUnfilledQuantityIndex = 17; + public const int TradeQuantityIndex = 18; + public const int AvgTradePriceIndex = 19; + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BrandonChen/BboPriceSummaryStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BrandonChen/BboPriceSummaryStream.cs new file mode 100644 index 000000000..1f0813843 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BrandonChen/BboPriceSummaryStream.cs @@ -0,0 +1,169 @@ +namespace bts.BlueOrb; + +using BCG; + +using bts.api; +using bts.collections; + +using Bts.DataStreamCore; +using Bts.PubSub; +using Bts.SharedStreams.StreamFactories; + +using Spark.Core; +using Spark.Core.Collections; +using Spark.Core.Disposables; + +using System.Collections.Immutable; +using System.Diagnostics; + +[DataStreamUserRespresentation(false)] +public class BboPriceSummaryStream : ViewStream, IDisposable { + readonly IObservable bboStream; + readonly IObservable tradeStream; + readonly IWorkScheduler workScheduler; + readonly TimeSpan publishInterval; + IBlueRowCollection? lastFired; + + ImmutableSortedSet> bookTimeseries = + ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<(DateTime, int), BookRowData>(TupleComparer.Comparer())); + ImmutableSortedSet> tradeTimeseries = + ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer<(DateTime, int), TradeRowData>(TupleComparer.Comparer())); + + int isDirty = 0; + IDisposable? sub; + + public BboPriceSummaryStream(IObservable bboStream, IObservable tradeStream, IWorkScheduler workScheduler, TimeSpan publishInterval) { + this.bboStream = bboStream; + this.tradeStream = tradeStream; + this.workScheduler = workScheduler; + this.publishInterval = publishInterval; + } + + ~BboPriceSummaryStream() => + Dispose(); + + public IEnumerable GetRows() { + int idCounter = 1; + var rows = bookTimeseries.AggregateConsecutiveGroups( + () => new BboPriceSummaryRow(idCounter++), + (r, kv) => r.OnBook(kv.Value), + kv => (kv.Value.BidPrice, kv.Value.AskPrice)) + .ToList(); + var tradeList = tradeTimeseries.ToList(); + int tradePos = 0; + foreach (var (r, next) in rows.PairWithPrevious()) { + if (r is null) { + continue; + } + var rOut = r; + if (rOut.BidTime.HasValue || rOut.AskTime.HasValue) { + var openTime = r.OpenTimestamp.assert_value(); + var closeTime = next.OpenTimestamp.assert_value(); + while (tradePos < tradeList.Count) { + var trade = tradeList[tradePos].Value; + if (trade.Timestamp < openTime) { + continue; + } + if (trade.Timestamp >= closeTime) { + break; + } + rOut = rOut.OnTrade(tradeList[tradePos].Value); + tradePos++; + } + } + yield return rOut; + } + } + + public void Dispose() { + GC.SuppressFinalize(this); + Disposable.dispose(ref sub); + } + + protected override void myOnFirstSubscriber() { + sub = Disposable.Create( + bboStream.Subscribe(OnBbo), + tradeStream.Subscribe(OnTrade), + workScheduler.Subscribe(Publish, publishInterval)); + + void OnBbo(IDataPacket dp) { + var bookTimeseriesBd = bookTimeseries.ToBuilder(); + foreach (var r in dp.deletedRows) { + var rd = new BookRowData(r); + Log.assert(bookTimeseriesBd.Remove(new((rd.Timestamp, rd.Id), default!))); + } + foreach (var r in dp.addedRows) { + var rd = new BookRowData(r); + Log.assert(bookTimeseriesBd.Add(new((rd.Timestamp, rd.Id), rd))); + } + bookTimeseries = bookTimeseriesBd.ToImmutable(); + Volatile.Write(ref isDirty, 1); + } + + void OnTrade(IDataPacket dp) { + var tradeTimeseriesBd = tradeTimeseries.ToBuilder(); + foreach (var r in dp.deletedRows) { + var rd = new TradeRowData(r); + Log.assert(tradeTimeseriesBd.Remove(new((rd.Timestamp, rd.Id), rd))); + } + foreach (var r in dp.addedRows) { + var rd = new TradeRowData(r); + Log.assert(tradeTimeseriesBd.Add(new((rd.Timestamp, rd.Id), rd))); + } + tradeTimeseries = tradeTimeseriesBd.ToImmutable(); + Volatile.Write(ref isDirty, 1); + } + + void Publish() { + if (Interlocked.CompareExchange(ref isDirty, 0, 1) == 0) { + return; + } + var sw = Stopwatch.StartNew(); + lastFired ??= new BlueTable(BboPriceSummaryRow.Schema); + var rows = GetRows().ToList(); + var table = new BlueTableListWrapper(BboPriceSummaryRow.Schema, rows.WrapSelector(r => (IBlueRow)r, null)); + var removes = new BlueTableListWrapper(BboPriceSummaryRow.Schema, lastFired.Where(r => !HasRow(table, r)).ToList()); + var adds = new BlueTableListWrapper(BboPriceSummaryRow.Schema, table.Where(r => !HasRow(lastFired, r)).ToList()); + lastFired = table; + sw.Stop(); + if (sw.Elapsed > TimeSpan.FromMilliseconds(20)) { + Log.DataFlow.warn($$"""[bbo_price_change_stream_slow_pub.json]{ "dur": "{{sw.ElapsedMilliseconds}}ms", "num_books": {{bookTimeseries.Count}} "num_trades": {{tradeTimeseries.Count}}, "ts": "{{DateTime.Now}}"}"""); + } + myFireEvent(new DataPacket(table, adds, removes)); + + bool HasRow(IBlueRowCollection? rows, IBlueRow row) { + if (rows is null) { + return false; + } + var id = row.Id(); + return rows.containsID(id) && BlueRowEqualityComparer.Equals(rows[id], row); + } + } + } + + protected override void myOnLastSubscriber() => + Disposable.dispose(ref sub); + + class BookSideRowData { + public BookSideRowData(Side side, IBlueRow row) { + Side = side; + Row = row; + Id = row.Id(); + Timestamp = row.GetDateTime(WellKnownColumns.Timestamp.Name).assert_value(); + var (priceCol, qtyCol) = side switch { + Side.Buy => (WellKnownColumns.BidPrice.Name, WellKnownColumns.BidQuantity.Name), + Side.Sell => (WellKnownColumns.AskPrice.Name, WellKnownColumns.AskQuantity.Name), + _ => throw new ArgumentException("Invalid side", nameof(side)), + }; + Price = row.GetFloat64(priceCol).assert_value(); + Quantity = row.GetFloat64(qtyCol).assert_value(); + } + + public Side Side { get; } + public IBlueRow Row { get; } + public int Id { get; } + public DateTime Timestamp { get; } + public double Price { get; } + public double Quantity { get; } + } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BrandonChen/BboRowData.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BrandonChen/BboRowData.cs new file mode 100644 index 000000000..91fbd21fd --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BrandonChen/BboRowData.cs @@ -0,0 +1,28 @@ +namespace bts.BlueOrb; + +using BCG; + +using Bts.DataStreamCore; +using Bts.SharedStreams.StreamFactories; + +using Spark.Core; + +public class BookRowData { + public BookRowData(IBlueRow row) { + Row = row; + Id = row.Id(); + Timestamp = row.GetDateTime(WellKnownColumns.Timestamp.Name).assert_value(); + BidPrice = row.GetFloat64(WellKnownColumns.BidPrice.Name).assert_value(); + BidQuantity = row.GetFloat64(WellKnownColumns.BidQuantity.Name).assert_value(); + AskPrice = row.GetFloat64(WellKnownColumns.AskPrice.Name).assert_value(); + AskQuantity = row.GetFloat64(WellKnownColumns.AskQuantity.Name).assert_value(); + } + + public IBlueRow Row { get; } + public int Id { get; } + public DateTime Timestamp { get; } + public double BidPrice { get; } + public double BidQuantity { get; } + public double AskPrice { get; } + public double AskQuantity { get; } +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BrandonChen/TradeRowData.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BrandonChen/TradeRowData.cs new file mode 100644 index 000000000..8ea5c591b --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/BrandonChen/TradeRowData.cs @@ -0,0 +1,28 @@ +namespace bts.BlueOrb; + +using BCG; + +using bts.co2; + +using Bts.DataStreamCore; +using Bts.SharedStreams.StreamFactories; + +using Spark.Core; + +public class TradeRowData { + public TradeRowData(IBlueRow row) { + Row = row; + Id = row.Id(); + Timestamp = row.GetDateTime(WellKnownColumns.Timestamp.Name).assert_value(); + AggressorSide = row.GetEnum("aggressor_side") ?? AggressorSide.None; + Price = row.GetFloat64(WellKnownColumns.Price.Name).assert_value(); + Quantity = row.GetFloat64(WellKnownColumns.Quantity.Name).assert_value(); + } + + public IBlueRow Row { get; } + public int Id { get; } + public DateTime Timestamp { get; } + public AggressorSide AggressorSide { get; } + public double Price { get; } + public double Quantity { get; } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/BookStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/BookStreamFactory.cs index 5e7e02f4b..cc5829904 100644 --- a/src/DataStreamSuite/SharedStreams/StreamFactories/BookStreamFactory.cs +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/BookStreamFactory.cs @@ -2,38 +2,90 @@ namespace Bts.SharedStreams.StreamFactories; using BCG; +using bts.BlueOrb; +using bts.collections; + using Bts.DataStream; using Bts.DataStreamCore; using Bts.DataStreamCore.StreamFactories; +using Bts.PubSub; using Microsoft.Extensions.Logging; using Spark.Api.MarketData; +using Spark.Core.Reactive; using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Reactive.Linq; public class BookStreamFactory : DataStreamFactory, IRecursiveDataStreamFactory { readonly LiveBboStreamFactory liveBboFactory; + readonly IMarketDataClient? mdClient; + readonly ILogger logger; + readonly IWorkScheduler workScheduler; IDataStreamFactory? compositeFactory = null; - public BookStreamFactory(IMarketDataClient? mdClient, ILogger logger) { - this.Add("book", CreateBook, logger); - this.Add(new[] { "bbo" }, CreateBbo, logger); + public BookStreamFactory(IMarketDataClient? mdClient, ILogger logger, ViewStreamDatabase workScheduler) { + this.mdClient = mdClient; + this.logger = logger; + this.workScheduler = workScheduler; + this.Add(new[] { "book" }, CreateBook, logger, TryGetBookStreamParams); + this.Add(new[] { "bbo" }, CreateBbo, logger, TryGetBookStreamParams); + this.Add(new[] { "bboPriceSummary" }, createBboPriceSummary, logger, TryGetBookStreamParams); liveBboFactory = new LiveBboStreamFactory(mdClient, logger); Add(liveBboFactory); - ViewStream? CreateBook(BookStreamParams streamParams) => - mdClient?.Books(streamParams.GetSecurityCriteria()) - ?.AsDataPackets(streamParams.GetSchemaBuilder().BookTupleAdapter()) - ?.AsViewStream() - ?? throw new InvalidOperationException("Live md not available"); + bool TryGetBookStreamParams(IEnumerable<(string? Name, int? Index, object? Value)> args, [NotNullWhen(true)]out BookStreamParams? result, [NotNullWhen(false)]out string? errorMsg) { + var argList = args.ToList(); + if (argList.Count == 0) { + errorMsg = "Missing required parameters (symbol)"; + result = null; + return false; + } + if (!StreamFactoryArgStore.TryGetParams(args, out result, out errorMsg)) { + result = new(); + } + var mdArgStore = new MarketDataArgStore(argList, "bboPriceSummary", logger); + if (!result.Venue.HasValue) { + result.Venue = mdArgStore.GetVenue(required: false); + } + if (string.IsNullOrEmpty(result.Symbol)) { + result.Symbol = mdArgStore.GetSymbol(required: false); + } + if (string.IsNullOrEmpty(result.SecurityName)) { + result.SecurityName = mdArgStore.GetSecurityName(required: false); + } + if (!result.SecurityType.HasValue) { + result.SecurityType = mdArgStore.GetSecurityType(required: false); + } + if (string.IsNullOrEmpty(result.Symbol) && string.IsNullOrEmpty(result.SecurityName)) { + errorMsg = "Missing required parameters (symbol OR securityName)"; + return false; + } + errorMsg = null; + return true; + } + + ViewStream? CreateBook(BookStreamParams streamParams) { + if (!checkHasLiveMd()) { + return null; + } + return this.mdClient.Books(streamParams.GetSecurityCriteria()) + .AsDataPackets(streamParams.GetSchemaBuilder().BookTupleAdapter()) + .AsViewStream() + .TimeSeries(streamParams.BufferSize, WellKnownColumnNames.Timestamp); + } ViewStream? CreateBbo(BookStreamParams streamParams) { - return mdClient?.Books(streamParams.GetSecurityCriteria()) - ?.AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter()) - ?.AsViewStream() - ?? throw new InvalidOperationException("Live md not available"); + if (!checkHasLiveMd()) { + return null; + } + return this.mdClient.Books(streamParams.GetSecurityCriteria()) + .AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter()) + .AsViewStream() + .TimeSeries(streamParams.BufferSize, WellKnownColumnNames.Timestamp); } } @@ -42,6 +94,46 @@ public class BookStreamFactory : DataStreamFactory, IRecursiveDataStreamFactory liveBboFactory.OnCompositeFactory(compositeFactory); } + ViewStream? createBboPriceSummary(BookStreamParams streamParams) { + if (!checkHasLiveMd()) { + return null; + } + var secCriteria = streamParams.GetSecurityCriteria(); + mdClient.Books(secCriteria).WithHistory().Select(t => t.Table); + // var bboStream = mdClient.Books(secCriteria).AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter()); + var bboStream = liveBboFactory.Create(streamParams) + ?? throw new ArgumentException("Failed to create bbo stream", nameof(streamParams)); + var bboTimeSeriesStream = new TimeSeriesStream(bboStream, streamParams.BufferSize, WellKnownColumnNames.Timestamp); + ViewStream? tradeStream = null; + if (streamParams.Symbol == "ES" && compositeFactory is not null) { + tradeStream = compositeFactory.Create("ReplayFutureLiveTradeStream", new (string?, int?, object?)[] { ("contract", 0, streamParams.Symbol) }); + if (tradeStream is null) { + logger.LogError("Failed to create trade stream for bboPriceSummary stream"); + return null; + } + tradeStream = tradeStream.RenameColumns(true, ("trade_price", WellKnownColumnNames.Price), ("trade_size", WellKnownColumnNames.Quantity)); + } else { + var tradeSchemaBd = new TradeSchemaBuilder(); + var tradeSchema = tradeSchemaBd.ToSchema(); + var tradeAdapter = tradeSchemaBd.TradeInfoAdapter(); + tradeStream = TradeStreamFactory.TradeStream(mdClient.Trades(secCriteria), tradeSchema, tradeAdapter).AsViewStream(); + } + var bboPriceSummaryStream = new BboPriceSummaryStream(bboStream.AsObservable(), tradeStream.AsObservable(), workScheduler, TimeSpan.FromSeconds(1)); + return bboPriceSummaryStream; + } + + static bool checkHasLiveMd([NotNullWhen(true)] IMarketDataClient? mdClient, ILogger? logger = null) { + if (mdClient is not null) { + return true; + } + logger?.LogError("Market data is unavailable (md client is null)"); + return false; + } + + [MemberNotNullWhen(true, nameof(mdClient))] + bool checkHasLiveMd() => + checkHasLiveMd(mdClient, logger); + class LiveBboStreamFactory : IDataStreamFactory, IRecursiveDataStreamFactory { static readonly IDataStreamMetadata metadata = new DataStreamMetadata() { Name = "LiveBBOStream", @@ -110,9 +202,26 @@ public class BookStreamFactory : DataStreamFactory, IRecursiveDataStreamFactory } } + public ViewStream? Create(BookStreamParams streamParams) { + var secType = streamParams.SecurityType; + if (streamParams.Symbol == "ES" && (!secType.HasValue || secType.Value == bts.api.SecurityType.Future) && compositeFactory is not null) { + return compositeFactory.Create("ReplayFutureLiveBBOStream", new (string?, int?, object?)[] { ("contract", 0, streamParams.Symbol) }); + } + if (!checkHasLiveMd()) { + return null; + } + return mdClient.Books(streamParams.GetSecurityCriteria()) + .AsDataPackets(streamParams.GetSchemaBuilder().BboAdapter()) + .AsViewStream(); + } + public IEnumerable GetSupportedStreams() { yield return metadata; } + + [MemberNotNullWhen(true, nameof(mdClient))] + bool checkHasLiveMd() => + BookStreamFactory.checkHasLiveMd(mdClient, logger); } public class LiveBboStreamParams { diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/BookStreamParams.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/BookStreamParams.cs index e1a8e9018..2c0268e88 100644 --- a/src/DataStreamSuite/SharedStreams/StreamFactories/BookStreamParams.cs +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/BookStreamParams.cs @@ -32,6 +32,9 @@ public class BookStreamParams { [StreamArgument(RegexNames = new[] { "$(show|include)?(cashPerPoint|cpp)^" })] public bool ShowCashPerPoint { get; set; } = true; + [StreamArgument(RegexNames = new[] { "$(buffer|size|bufferSize|cap|capacity|len|length)^" })] + public int BufferSize { get; set; } = 10_000; + public mbts.spark.SecurityCriteria GetSecurityCriteria() { var clause = new mbts.spark.SecurityCriteriaClause( exchanges: Venue.HasValue ? new[] { Venue.Value } : null, diff --git a/src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs b/src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs index 008eeb3cb..3cfcc9042 100644 --- a/src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs +++ b/src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs @@ -327,6 +327,21 @@ namespace System.Collections.Generic { } } + public static IEnumerable<(T? Previous, T Current)> PairWithPrevious(this IEnumerable source, bool includeZeroIndex = true) { + using var en = source.GetEnumerator(); + if (!en.MoveNext()) { + yield break; + } + var prev = en.Current; + if (includeZeroIndex) { + yield return (default, prev); + } + while (en.MoveNext()) { + yield return (prev, en.Current); + prev = en.Current; + } + } + /// /// Checks if implements or . If so, the resultant Count is returned, otherwise null (will not enumerate through any items). /// @@ -486,6 +501,19 @@ namespace System.Collections.Generic { return false; } + public static bool TryGetLast(this IEnumerable source, [MaybeNullWhen(false)]out T result) { + ArgumentNullException.ThrowIfNull(source); + using var en = source.GetEnumerator(); + if (!en.MoveNext()) { + result = default; + return false; + } + do { + result = en.Current; + } while (en.MoveNext()); + return true; + } + public static bool TryGetSingle(this IEnumerable source, out T? result) { if (source == null) throw new ArgumentNullException(nameof(source)); using (var en = source.GetEnumerator()) { diff --git a/src/client/spark_core/Update/TableUpdateWithHistory.cs b/src/client/spark_core/Update/TableUpdateWithHistory.cs new file mode 100644 index 000000000..075b74bc6 --- /dev/null +++ b/src/client/spark_core/Update/TableUpdateWithHistory.cs @@ -0,0 +1,97 @@ +#nullable enable +namespace Spark.Core.Reactive; + +using bts; +using bts.api.core; + +using Spark.Core.Collections; + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Reactive.Linq; + +public static class TableUpdateWithHistory { + public const int DEFAULT_CAPACITY = 16; + + public static TableUpdateWithHistory Empty(int capacity = DEFAULT_CAPACITY) where TKey : IEquatable => + new(); + + public static IObservable> WithHistory(this IObservable> source) where TKey : IEquatable { + return source.Scan(Empty(), AddHistory); + + TableUpdateWithHistory AddHistory(TableUpdateWithHistory history, TableUpdate update) => + history.Update(update); + } +} + +public class TableUpdateWithHistory where TKey : IEquatable { + readonly CircularBuffer> buffer; + readonly int capacity; + readonly bool dropOnRemove; + readonly TableUpdate table = TableUpdate.mk_empty(r => r.Key); + readonly long version; + + public TableUpdateWithHistory(int capacity = TableUpdateWithHistory.DEFAULT_CAPACITY, bool dropOnRemove = true) + : this(TableUpdate.mk_empty(r => r.Key), 0, capacity, dropOnRemove) { } + + TableUpdateWithHistory(TableUpdate table, long version, int capacity, bool dropOnRemove = true) { + buffer = new(capacity); + this.capacity = capacity; + this.version = version; + this.dropOnRemove = dropOnRemove; + } + + public TableUpdate Table => table; + + public TableUpdateWithHistory Update(TableUpdate upd) { + var getKey = Log.notnull(upd.key_extractor); + long version = this.version + 1; + buffer.Add(upd); + var bd = table.to_builder(); + var time = DateTime.Now; + if (dropOnRemove) { + foreach (var v in upd.removed) { + bd.remove(getKey(v)); + } + } + foreach (var v in upd.changed.Concat(upd.added)) { + var k = getKey(v); + if (!bd.TryGetValue(k, out var rec)) { + rec = new(k); + } + rec = rec.Update(v, time, version, capacity); + bd.add_or_update(rec); + } + return new(bd.to_table(), version, capacity, dropOnRemove); + } + + public readonly struct Record { + public readonly ImmutableList<(TValue Value, DateTime Time, long UpdateNum)>? Buffer; + + public Record(TKey key, ImmutableList<(TValue Value, DateTime Time, long UpdateNum)>? buffer = null) { + Key = key; + Buffer = buffer; + } + + public TKey Key { get; } + + [MemberNotNullWhen(false, nameof(Buffer))] + public bool IsEmpty => Count == 0; + + public int Count => Buffer?.Count ?? 0; + public TValue Value => Buffer?.TryGetLast(out var lst) ?? false + ? lst.Value + : throw new InvalidOperationException("Buffer is empty"); + + public Record Update(TValue value, DateTime time, long updateNum, int capacity) { + var buffer = Buffer ?? ImmutableList<(TValue Value, DateTime Time, long UpdateNum)>.Empty.Add((value, time, updateNum)); + if (buffer.Count >= capacity) { + buffer = buffer.RemoveRange(0, buffer.Count - capacity + 1); + } + return new(Key, buffer); + } + } +}