From c839652ea3c286ab98d32da01b14816f68e79ae0 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Fri, 21 Nov 2025 10:34:36 -0500 Subject: [PATCH] refactor: Make CircularBufferTable immutable --- .../DataStream/BlueOrb/MillisecondStream.cs | 2 +- .../DataStream/BlueOrb/TimeSeriesStream.cs | 62 +++++++++---- .../Primitives/Misc/SnapshotReplayStream.cs | 4 +- .../Primitives/Time/MillisecondStream.cs | 2 +- .../BlueData/BlueRowEqualityComparer.cs | 7 +- .../BlueData/CircularBufferTable.cs | 92 +++++++++++++------ 6 files changed, 118 insertions(+), 51 deletions(-) diff --git a/src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs index 4446ccc6a..2614e3447 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs @@ -43,7 +43,7 @@ namespace bts.BlueOrb { _last_fire_time = DateTime.Now; IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time, _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff")); - _table.add_row(nr); + _table = _table.Add(nr); var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket); myFireEvent(dp); } diff --git a/src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs index 557da7ea7..b93f8cfa5 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs @@ -1,25 +1,39 @@ -using System; -using System.Data.Entity.Core.Mapping; - using BCG; +using bts.utils; + +using Bts.DataStreamCore; + +using mbts.utils; + +using Spark.Core.Collections; +using Spark.Core.Threading; + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; + namespace bts.BlueOrb { // Keeps adding changing rows to the stream, hence preserving the history // and creating a time series, for example this can be used to make bbo updates // into a time series of prices/qty [DataStreamUserRespresentation("TimeSeriesStream")] public class TimeSeriesStream : ViewStream { - private CircularBufferTable _table; - private ViewStream _stream; + readonly ViewStream _stream; + readonly string timeColumn; + readonly AtomicGate gate = new(); + CircularBufferTable? _table; private int _id = 0; private int _size = 0; public TimeSeriesStream(ViewStream stream) : this(stream, Int32.MaxValue) { } - public TimeSeriesStream(ViewStream stream, int size) { + public TimeSeriesStream(ViewStream stream, int size, string timeColumn = "ts_event") { _stream = stream; _size = size; + this.timeColumn = timeColumn; } protected override void myOnFirstSubscriber() { @@ -31,27 +45,37 @@ namespace bts.BlueOrb { private void _onstream_change(object o) { - DataPacket dp = o as DataPacket; + var dp = ExceptionAssert.Argument.Is(o); DateTime now = DateTime.Now; + using var _ = gate.BeginTransaction(); if (_table == null) { - IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName); - schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp")); - _table = new CircularBufferTable(schema, _size); + var colInfos = dp.newRows.schema.columnInfo; + if (colInfos.TryGetFirst(c => c.colName == timeColumn, out var currTimeCol, out var currTimeColIdx)) { + colInfos = colInfos.ToArray(); + colInfos[currTimeColIdx] = new(timeColumn, currTimeCol.colNo, typeof(DateTime)); + } else { + colInfos = ArrayHelper.add(colInfos, new(timeColumn, colInfos.Length, typeof(DateTime))); + } + _table = new CircularBufferTable(new BlueTableSchema(colInfos, dp.newRows.schema.tableName), _size); } + var addedRowsBd = new BlueTableImmutable(_table.schema).ToDataPacketBuilder(); + var deletedRowsBd = new BlueTableImmutable(_table.schema).ToDataPacketBuilder(); foreach (IBlueRow r in dp.addedRows) { MutableBlueRow ar = new MutableBlueRow(_table.schema, r); - ar.setValue("timestamp", now); - ar.setValue("id", _id++); - _table.add_row(ar); + ar.setValue(timeColumn, now); + int id = _id++; + ar.setValue("id", id); + _table = _table.Add(ar, out var oldRow); + addedRowsBd.AddOrUpdate(ar); + if (oldRow is not null) { + deletedRowsBd.AddOrUpdate(oldRow); + } } - var dp_out = BlueDataUtility.build_dp(_table, myLastValue as DataPacket); + var addedRows = addedRowsBd.ToImmutable(); + var deletedRows = deletedRowsBd.ToImmutable(); + var dp_out = new DataPacket(_table, addedRows, deletedRows); myFireEvent(dp_out); } } - - - } - - diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs index 0cf9d6c7d..8697bf23a 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs @@ -66,7 +66,7 @@ namespace bts { if (_in_rows.Count > 0) { // fire row IBlueRow curr_row = _in_rows.ElementAt(_in_row_ptr++); - _out_table.add_row(curr_row); + _out_table = _out_table.Add(curr_row); DataPacket dp = BlueDataUtility.build_dp_inc(_out_table, myLastValue == null ? null : myLastValue as DataPacket, false); myFireEvent(dp); @@ -74,4 +74,4 @@ namespace bts { } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs index 82e745c3c..61d3bb966 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs @@ -44,7 +44,7 @@ namespace BCG { _last_fire_time = DateTime.Now; IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time, _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff")); - _table.add_row(nr); + _table = _table.Add(nr); var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket); myFireEvent(dp); } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowEqualityComparer.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowEqualityComparer.cs index 25929cb48..0f462c717 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowEqualityComparer.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowEqualityComparer.cs @@ -8,12 +8,15 @@ using System.Diagnostics.CodeAnalysis; using System.Linq; public class BlueRowEqualityComparer : IEqualityComparer { - public static BlueRowEqualityComparer Instance { get; } = new(); + public static IEqualityComparer Instance { get; } = new BlueRowEqualityComparer(); - public bool Equals(IBlueRow x, IBlueRow y) => + public static bool Equals(IBlueRow x, IBlueRow y) => x.schema.columnInfo.SequenceEqual(y.schema.columnInfo) && x.Values.Cast().SequenceEqual(y.Values.Cast()); + bool IEqualityComparer.Equals(IBlueRow x, IBlueRow y) => + Equals(x, y); + public int GetHashCode([DisallowNull] IBlueRow obj) { var hash = new HashCode(); foreach (var v in obj.Values) { diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs index c1322be68..aaff17c94 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs @@ -1,53 +1,93 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Data; -using System.Linq; +using System.Collections.Immutable; using BCG; +using bts.collections; + +using Bts.DataStreamCore; + namespace bts { public class CircularBufferTable : IBlueRowCollection { - private BlueTable _table; - private List _ordered_list = new List(); + readonly ImmutableList orderedRows; + readonly ImmutableDictionary rowIndex; private int _buffer_sz; - public CircularBufferTable(IBlueTableSchema schema, int buffer_sz) { - _table = new BlueTable(schema); - _buffer_sz = buffer_sz; + public CircularBufferTable(IBlueTableSchema schema, int buffer_sz) + : this(schema, buffer_sz, ImmutableList.Empty, ImmutableDictionary.Empty) { } + + CircularBufferTable(IBlueTableSchema schema, int bufferSize, ImmutableList orderedRows, ImmutableDictionary rowIndex) { + this.schema = schema; + _buffer_sz = bufferSize; + this.orderedRows = orderedRows; + this.rowIndex = rowIndex; } + public CircularBufferTable Add(IBlueRow row) => + Add(row, out _); + // returns the row that might need to be deleted to keep // the circular buffer intact - public IBlueRow add_row(IBlueRow row) { - _ordered_list.Add(row); - _table.addRow(row); - if (_ordered_list.Count > _buffer_sz) { - IBlueRow rv = _ordered_list.ElementAt(0); - _ordered_list.RemoveAt(0); - _table.deleteRow(rv); - return rv; + public CircularBufferTable Add(IBlueRow row, out IBlueRow oldRow) { + int id = row.Id(); + var rowIndexBd = rowIndex.ToBuilder(); + var orderedRowsBd = orderedRows.ToBuilder(); + if (rowIndexBd.TryGetValue(id, out oldRow)) { + Log.assert(orderedRowsBd.RemoveFirst(r => r.Id() == id)); + } else if (rowIndexBd.Count > _buffer_sz) { + oldRow = orderedRowsBd[0]; + rowIndexBd.Remove(oldRow.Id()); + orderedRowsBd.RemoveAt(0); } - return null; + rowIndexBd[id] = row; + orderedRowsBd.Add(row); + return new(schema, _buffer_sz, orderedRowsBd.ToImmutable(), rowIndexBd.ToImmutable()); } public IBlueRow this[int id] { get { - return _table[id]; - } - } - public IBlueTableSchema schema { - get { - return _table.schema; + return orderedRows[id]; } } + public IBlueTableSchema schema { get; } public bool containsID(int id) { - return _table.containsID(id); + return rowIndex.ContainsKey(id); } - public void CopyTo(Array array, int index) { _table.CopyTo((IBlueRow[])array, index); } - public int Count { get { return _table.Count; } } + public void CopyTo(Array array, int index) { + foreach (var row in orderedRows) { + array.SetValue(row, index++); + } + } + public int Count => rowIndex.Count; public bool IsSynchronized { get { return false; } } public object SyncRoot { get { return this; } } - public System.Collections.IEnumerator GetEnumerator() { return _table.GetEnumerator(); } + public IEnumerator GetEnumerator() => + orderedRows.GetEnumerator(); + + public IList GetColumn(int index) => + orderedRows.WrapSelector(r => r[index], null); + + public IBlueRow GetRow(int index) => + orderedRows[index]; + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + if (rowIndex.TryGetValue(id, out var row) && BlueRowEqualityComparer.Equals(item, row)) { + int i = 0; + foreach (var r in orderedRows) { + if (r.Id() == id) { + return i; + } + i++; + } + } + return -1; + } + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); } } +