diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByEvaluator.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByEvaluator.cs index 0802b91e5..7234c430f 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByEvaluator.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByEvaluator.cs @@ -69,14 +69,7 @@ namespace BCG protected IBlueRow myProject (IBlueRow r) {return myProject (r, r == null ? Int32.MinValue : (int)r["id"]);} #endregion - private IBlueRow myProjectRow (IBlueRow r, string colMapExpression) - { - BlueTable auxTable = new BlueTable (r.schema); - auxTable.addRow (r); - BlueRowCollectionStaticStream staticStream = new BlueRowCollectionStaticStream (auxTable); - ProjectStream projStream = new ProjectStream (staticStream, colMapExpression); - return BlueDataUtility.extractRow (projStream.rows, 0); - } - + IBlueRow myProjectRow (IBlueRow r, string colMapExpression) => + new ProjectedRow(r, new ProjectHelper(r.schema, colMapExpression, "GroupByEvaluator_Project")); } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/Proximity/ProximityJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/Proximity/ProximityJoinStream.cs index b9c136435..568314b82 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/Proximity/ProximityJoinStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/Proximity/ProximityJoinStream.cs @@ -2,11 +2,13 @@ namespace bts { using BCG; using BCG.Utilities.General; + using mbts.utils; + + using Spark.Core.Disposables; + using System; using System.Diagnostics; - using ZstdSharp.Unsafe; - // functon that joins the 2 rows and returns the outputput row. The function // can return a null which means the 2 rows dont need to be joined using JoinFunc = System.Func; @@ -16,6 +18,10 @@ namespace bts { private bool _dirty = false; private Stopwatch _stopwatch = new Stopwatch(); private ViewStreamDatabase _vsd = ViewStreamDatabase.instance; + IDisposable? subscription; + IBlueRowCollection? leftRows; + IBlueRowCollection? rightRows; + public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) { _lvs = lvs; _rvs = rvs; @@ -23,31 +29,44 @@ namespace bts { } protected override void myOnFirstSubscriber() { - _lvs.onChange += _handle_data; - _rvs.onChange += _handle_data; + _lvs.onChange += OnLeftChange; + _rvs.onChange += OnRightChange; + subscription = Disposable.Create(Unsubscribe); _vsd.addWorker(this); + + void OnLeftChange(object dp_obj) { + leftRows = ExceptionAssert.Argument.Is(dp_obj).newRows; + _dirty = true; + doWork(); + } + + void OnRightChange(object dp_obj) { + rightRows = ExceptionAssert.Argument.Is(dp_obj).newRows; + _dirty = true; + doWork(); + } + + void Unsubscribe() { + _lvs.onChange -= OnLeftChange; + _rvs.onChange -= OnRightChange; + } } protected override void myOnLastSubscriber() { - _lvs.onChange -= _handle_data; - _rvs.onChange -= _handle_data; + subscription?.Dispose(); + subscription = null; _vsd.removeWorker(this); } - private void _handle_data(object dp_obj) { - _dirty = true; - doWork(); - } - public void doWork() { - if (!_dirty || _lvs.rows is null || _rvs.rows is null) { + if (!_dirty || leftRows is null || rightRows is null) { return; } _stopwatch.Start(); _dirty = false; BlueTable? rv = null; - foreach (IBlueRow lr in _lvs.rows) { - foreach (IBlueRow rr in _rvs.rows) { + foreach (IBlueRow lr in leftRows) { + foreach (IBlueRow rr in rightRows) { IBlueRow resr = _join_func(lr, rr); if (resr == null) continue; @@ -56,7 +75,7 @@ namespace bts { } } _stopwatch.Stop(); - Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv?.Count ?? 0} time: {_stopwatch.ElapsedMilliseconds} msecs"); + Log.Gen.info($"ProximityJoinStream left_count: {leftRows.Count} right_count: {rightRows.Count} result_count: {rv?.Count ?? 0} time: {_stopwatch.ElapsedMilliseconds} msecs"); var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket); myFireEvent(dp); } diff --git a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs index aa132b1b2..cc40e5b12 100644 --- a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs @@ -111,16 +111,21 @@ namespace BCG { _file_address = fileAddress; } + public static IBlueRowCollection LoadFile(string path) { + var sw = Stopwatch.StartNew(); + bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber opening address{path}"); + var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(path); + sw.Stop(); + bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {sw.ElapsedMilliseconds} msecs"); + return rows; + } + protected override void myOnFirstSubscriber() { //_stopwatch.Start(); //DataFrame df = DataFrame.LoadCsv(_file_address); //_stopwatch.Stop(); //bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs"); - _stopwatch.Start(); - bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber opening address{_file_address}"); - IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address); - _stopwatch.Stop(); - bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs"); + var rows = LoadFile(_file_address); DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket); myFireEvent(dp_out); } diff --git a/src/DataStreamSuite/DataStream/Primitives/StaticCollectionStream/BlueTableStream.cs b/src/DataStreamSuite/DataStream/Primitives/StaticCollectionStream/BlueTableStream.cs index c2a458a07..96ade131c 100644 --- a/src/DataStreamSuite/DataStream/Primitives/StaticCollectionStream/BlueTableStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/StaticCollectionStream/BlueTableStream.cs @@ -36,24 +36,6 @@ namespace BCG { } public void loadPacket(DataPacket dp) { myFireEvent(dp); } - - /// - /// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc - /// The other arguments are provided by the user. - /// - /// - /// - /// - /// - public static IBlueRowCollection statelessOp(IBlueRowCollection rows, Type streamType, params object[] args) { - ArrayList totArgs = new ArrayList(args); - BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream(); - totArgs.Insert(0, v1); - ViewStream retVal = Activator.CreateInstance(streamType, totArgs.ToArray()) as ViewStream; - v1.loadCollection(rows); - return retVal.rows; - } - } diff --git a/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextWriterStream.cs b/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextWriterStream.cs index 3b49e5d55..445bfb35e 100644 --- a/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextWriterStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextWriterStream.cs @@ -4,11 +4,16 @@ using System.IO; using BCG.Utilities.General; +using bts; + +using mbts.utils; + namespace BCG { [DataStreamUserRespresentation("textWriter")] public class TextWriterStream : ViewStream, IWorker { + IBlueRowCollection? rows; #region constructor public TextWriterStream(ViewStream stream, string fileNamePrefix, int writeFreqSecs) : this(ViewStreamDatabase.instance, stream, fileNamePrefix, writeFreqSecs) @@ -53,13 +58,14 @@ namespace BCG { myGotUpdate = false; myNextWrite = DateTime.Now.AddSeconds(myWriteFreqSecs); - myWrite(myStream.rows); + myWrite(Log.notnull(rows)); } } #endregion private void myStreamChange(object o) { + rows = ExceptionAssert.Argument.Is(o).newRows; myGotUpdate = true; } diff --git a/src/DataStreamSuite/DataStreamCore/ViewStream.cs b/src/DataStreamSuite/DataStreamCore/ViewStream.cs index aa9bc98ac..14e7b782a 100644 --- a/src/DataStreamSuite/DataStreamCore/ViewStream.cs +++ b/src/DataStreamSuite/DataStreamCore/ViewStream.cs @@ -32,37 +32,6 @@ namespace BCG ?? (myRealSource as ViewStream)?.StreamName ?? GetType().Name; - public IBlueRowCollection rows - { - get - { - if (myRealSource != null) - { - // BlueEventLog.logAssert (myRealSource is ViewStream); - - return (myRealSource as ViewStream).rows; - } - else - { - // used to force a computation of a stream which - // happens to have no subscribers - onChange += new DataStream.ChangeDelegate(myDummy); - try - { - - if (myLastValue != null) - return ((DataPacket) myLastValue).newRows; - else - return null; - } - finally - { - onChange -= new DataStream.ChangeDelegate(myDummy); - } - } - } - } - protected override object myGetValueForInitialSubscription() { if (myLastValue == null) diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveBBOStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveBBOStream.cs index 4f3e6de34..a8d67d959 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveBBOStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveBBOStream.cs @@ -63,8 +63,9 @@ protected override void myOnFirstSubscriber() { var file_path = Path.Combine(SQLiteHelper.location, $"mbp1_{_fcc.symbol}_small.csv".ToLower()); - var csv_rd = new CSVReaderStream(file_path, false); - _rows.rows = csv_rd.rows.to_list().OrderBy(r => r.get("ts_event")).ToList(); + var rows = BCG.CSVReaderStream.LoadFile(file_path); + _rows.rows = new(rows.Count); + _rows.rows.AddRange(rows.OfType().OrderBy(r => r.get("ts_event"))); _rows.pointer = 0; _vsd.addWorker(this); } diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveTradeStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveTradeStream.cs index 2cda5019d..6d5a135f3 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveTradeStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveTradeStream.cs @@ -63,7 +63,9 @@ protected override void myOnFirstSubscriber() { var file_path = Path.Combine(SQLiteHelper.location, $"trades_{_fcc.symbol}.csv"); var csv_rd = new CSVReaderStream(file_path, false); - _rows.rows = csv_rd.rows.to_list().OrderBy(r => r.get("ts_event")).ToList(); + var rows = BCG.CSVReaderStream.LoadFile(file_path); + _rows.rows = new(rows.Count); + _rows.rows.AddRange(rows.OfType().OrderBy(r => r.get("ts_event"))); _rows.pointer = 0; _vsd.addWorker(this); }