refactor: Get rid of ViewStream.rows prop

It was causing weird side effects in debugger
This commit is contained in:
John Colvin committed 2026-01-10 12:59:14 -05:00
1 parent 327665b4c3
commit bb5ea19f69
8 files changed
+59 -82

No files matched your search

@@ -69,14 +69,7 @@ namespace BCG
protected IBlueRow myProject (IBlueRow r) {return myProject (r, r == null ? Int32.MinValue : (int)r["id"]);}
#endregion
private IBlueRow myProjectRow (IBlueRow r, string colMapExpression)
{
BlueTable auxTable = new BlueTable (r.schema);
auxTable.addRow (r);
BlueRowCollectionStaticStream staticStream = new BlueRowCollectionStaticStream (auxTable);
ProjectStream projStream = new ProjectStream (staticStream, colMapExpression);
return BlueDataUtility.extractRow (projStream.rows, 0);
}
IBlueRow myProjectRow (IBlueRow r, string colMapExpression) =>
new ProjectedRow(r, new ProjectHelper(r.schema, colMapExpression, "GroupByEvaluator_Project"));
}
}
@@ -2,11 +2,13 @@ namespace bts {
using BCG;
using BCG.Utilities.General;
using mbts.utils;
using Spark.Core.Disposables;
using System;
using System.Diagnostics;
using ZstdSharp.Unsafe;
// functon that joins the 2 rows and returns the outputput row. The function
// can return a null which means the 2 rows dont need to be joined
using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
@@ -16,6 +18,10 @@ namespace bts {
private bool _dirty = false;
private Stopwatch _stopwatch = new Stopwatch();
private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
IDisposable? subscription;
IBlueRowCollection? leftRows;
IBlueRowCollection? rightRows;
public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
_lvs = lvs;
_rvs = rvs;
@@ -23,31 +29,44 @@ namespace bts {
}
protected override void myOnFirstSubscriber() {
_lvs.onChange += _handle_data;
_rvs.onChange += _handle_data;
_lvs.onChange += OnLeftChange;
_rvs.onChange += OnRightChange;
subscription = Disposable.Create(Unsubscribe);
_vsd.addWorker(this);
void OnLeftChange(object dp_obj) {
leftRows = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj).newRows;
_dirty = true;
doWork();
}
void OnRightChange(object dp_obj) {
rightRows = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj).newRows;
_dirty = true;
doWork();
}
void Unsubscribe() {
_lvs.onChange -= OnLeftChange;
_rvs.onChange -= OnRightChange;
}
}
protected override void myOnLastSubscriber() {
_lvs.onChange -= _handle_data;
_rvs.onChange -= _handle_data;
subscription?.Dispose();
subscription = null;
_vsd.removeWorker(this);
}
private void _handle_data(object dp_obj) {
_dirty = true;
doWork();
}
public void doWork() {
if (!_dirty || _lvs.rows is null || _rvs.rows is null) {
if (!_dirty || leftRows is null || rightRows is null) {
return;
}
_stopwatch.Start();
_dirty = false;
BlueTable? rv = null;
foreach (IBlueRow lr in _lvs.rows) {
foreach (IBlueRow rr in _rvs.rows) {
foreach (IBlueRow lr in leftRows) {
foreach (IBlueRow rr in rightRows) {
IBlueRow resr = _join_func(lr, rr);
if (resr == null)
continue;
@@ -56,7 +75,7 @@ namespace bts {
}
}
_stopwatch.Stop();
Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv?.Count ?? 0} time: {_stopwatch.ElapsedMilliseconds} msecs");
Log.Gen.info($"ProximityJoinStream left_count: {leftRows.Count} right_count: {rightRows.Count} result_count: {rv?.Count ?? 0} time: {_stopwatch.ElapsedMilliseconds} msecs");
var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
myFireEvent(dp);
}
@@ -111,16 +111,21 @@ namespace BCG {
_file_address = fileAddress;
}
public static IBlueRowCollection LoadFile(string path) {
var sw = Stopwatch.StartNew();
bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber opening address{path}");
var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(path);
sw.Stop();
bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {sw.ElapsedMilliseconds} msecs");
return rows;
}
protected override void myOnFirstSubscriber() {
//_stopwatch.Start();
//DataFrame df = DataFrame.LoadCsv(_file_address);
//_stopwatch.Stop();
//bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
_stopwatch.Start();
bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber opening address{_file_address}");
IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address);
_stopwatch.Stop();
bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
var rows = LoadFile(_file_address);
DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
myFireEvent(dp_out);
}
@@ -36,24 +36,6 @@ namespace BCG {
}
public void loadPacket(DataPacket dp) { myFireEvent(dp); }
/// <summary>
/// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc
/// The other arguments are provided by the user.
/// </summary>
/// <param name="rows"></param>
/// <param name="streamType"></param>
/// <param name="args"></param>
/// <returns></returns>
public static IBlueRowCollection statelessOp(IBlueRowCollection rows, Type streamType, params object[] args) {
ArrayList totArgs = new ArrayList(args);
BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream();
totArgs.Insert(0, v1);
ViewStream retVal = Activator.CreateInstance(streamType, totArgs.ToArray()) as ViewStream;
v1.loadCollection(rows);
return retVal.rows;
}
}
@@ -4,11 +4,16 @@ using System.IO;
using BCG.Utilities.General;
using bts;
using mbts.utils;
namespace BCG
{
[DataStreamUserRespresentation("textWriter")]
public class TextWriterStream : ViewStream, IWorker
{
IBlueRowCollection? rows;
#region constructor
public TextWriterStream(ViewStream stream, string fileNamePrefix, int writeFreqSecs)
: this(ViewStreamDatabase.instance, stream, fileNamePrefix, writeFreqSecs)
@@ -53,13 +58,14 @@ namespace BCG
{
myGotUpdate = false;
myNextWrite = DateTime.Now.AddSeconds(myWriteFreqSecs);
myWrite(myStream.rows);
myWrite(Log.notnull(rows));
}
}
#endregion
private void myStreamChange(object o)
{
rows = ExceptionAssert.Argument.Is<IDataPacket>(o).newRows;
myGotUpdate = true;
}
@@ -32,37 +32,6 @@ namespace BCG
?? (myRealSource as ViewStream)?.StreamName
?? GetType().Name;
public IBlueRowCollection rows
{
get
{
if (myRealSource != null)
{
// BlueEventLog.logAssert (myRealSource is ViewStream);
return (myRealSource as ViewStream).rows;
}
else
{
// used to force a computation of a stream which
// happens to have no subscribers
onChange += new DataStream.ChangeDelegate(myDummy);
try
{
if (myLastValue != null)
return ((DataPacket) myLastValue).newRows;
else
return null;
}
finally
{
onChange -= new DataStream.ChangeDelegate(myDummy);
}
}
}
}
protected override object myGetValueForInitialSubscription()
{
if (myLastValue == null)
@@ -63,8 +63,9 @@
protected override void myOnFirstSubscriber() {
var file_path = Path.Combine(SQLiteHelper.location, $"mbp1_{_fcc.symbol}_small.csv".ToLower());
var csv_rd = new CSVReaderStream(file_path, false);
_rows.rows = csv_rd.rows.to_list().OrderBy(r => r.get<string>("ts_event")).ToList();
var rows = BCG.CSVReaderStream.LoadFile(file_path);
_rows.rows = new(rows.Count);
_rows.rows.AddRange(rows.OfType<IBlueRow>().OrderBy(r => r.get<string>("ts_event")));
_rows.pointer = 0;
_vsd.addWorker(this);
}
@@ -63,7 +63,9 @@
protected override void myOnFirstSubscriber() {
var file_path = Path.Combine(SQLiteHelper.location, $"trades_{_fcc.symbol}.csv");
var csv_rd = new CSVReaderStream(file_path, false);
_rows.rows = csv_rd.rows.to_list().OrderBy(r => r.get<string>("ts_event")).ToList();
var rows = BCG.CSVReaderStream.LoadFile(file_path);
_rows.rows = new(rows.Count);
_rows.rows.AddRange(rows.OfType<IBlueRow>().OrderBy(r => r.get<string>("ts_event")));
_rows.pointer = 0;
_vsd.addWorker(this);
}