refactor: Get rid of ViewStream.rows prop
It was causing weird side effects in debugger
This commit is contained in:
1 parent
327665b4c3
commit
bb5ea19f69
8 files changed
+59
-82
No files matched your search
@@ -69,14 +69,7 @@ namespace BCG
|
||||
protected IBlueRow myProject (IBlueRow r) {return myProject (r, r == null ? Int32.MinValue : (int)r["id"]);}
|
||||
#endregion
|
||||
|
||||
private IBlueRow myProjectRow (IBlueRow r, string colMapExpression)
|
||||
{
|
||||
BlueTable auxTable = new BlueTable (r.schema);
|
||||
auxTable.addRow (r);
|
||||
BlueRowCollectionStaticStream staticStream = new BlueRowCollectionStaticStream (auxTable);
|
||||
ProjectStream projStream = new ProjectStream (staticStream, colMapExpression);
|
||||
return BlueDataUtility.extractRow (projStream.rows, 0);
|
||||
}
|
||||
|
||||
IBlueRow myProjectRow (IBlueRow r, string colMapExpression) =>
|
||||
new ProjectedRow(r, new ProjectHelper(r.schema, colMapExpression, "GroupByEvaluator_Project"));
|
||||
}
|
||||
}
|
||||
@@ -2,11 +2,13 @@ namespace bts {
|
||||
using BCG;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using Spark.Core.Disposables;
|
||||
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
|
||||
using ZstdSharp.Unsafe;
|
||||
|
||||
// functon that joins the 2 rows and returns the outputput row. The function
|
||||
// can return a null which means the 2 rows dont need to be joined
|
||||
using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
|
||||
@@ -16,6 +18,10 @@ namespace bts {
|
||||
private bool _dirty = false;
|
||||
private Stopwatch _stopwatch = new Stopwatch();
|
||||
private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
|
||||
IDisposable? subscription;
|
||||
IBlueRowCollection? leftRows;
|
||||
IBlueRowCollection? rightRows;
|
||||
|
||||
public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
|
||||
_lvs = lvs;
|
||||
_rvs = rvs;
|
||||
@@ -23,31 +29,44 @@ namespace bts {
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
_lvs.onChange += _handle_data;
|
||||
_rvs.onChange += _handle_data;
|
||||
_lvs.onChange += OnLeftChange;
|
||||
_rvs.onChange += OnRightChange;
|
||||
subscription = Disposable.Create(Unsubscribe);
|
||||
_vsd.addWorker(this);
|
||||
|
||||
void OnLeftChange(object dp_obj) {
|
||||
leftRows = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj).newRows;
|
||||
_dirty = true;
|
||||
doWork();
|
||||
}
|
||||
|
||||
void OnRightChange(object dp_obj) {
|
||||
rightRows = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj).newRows;
|
||||
_dirty = true;
|
||||
doWork();
|
||||
}
|
||||
|
||||
void Unsubscribe() {
|
||||
_lvs.onChange -= OnLeftChange;
|
||||
_rvs.onChange -= OnRightChange;
|
||||
}
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
_lvs.onChange -= _handle_data;
|
||||
_rvs.onChange -= _handle_data;
|
||||
subscription?.Dispose();
|
||||
subscription = null;
|
||||
_vsd.removeWorker(this);
|
||||
}
|
||||
|
||||
private void _handle_data(object dp_obj) {
|
||||
_dirty = true;
|
||||
doWork();
|
||||
}
|
||||
|
||||
public void doWork() {
|
||||
if (!_dirty || _lvs.rows is null || _rvs.rows is null) {
|
||||
if (!_dirty || leftRows is null || rightRows is null) {
|
||||
return;
|
||||
}
|
||||
_stopwatch.Start();
|
||||
_dirty = false;
|
||||
BlueTable? rv = null;
|
||||
foreach (IBlueRow lr in _lvs.rows) {
|
||||
foreach (IBlueRow rr in _rvs.rows) {
|
||||
foreach (IBlueRow lr in leftRows) {
|
||||
foreach (IBlueRow rr in rightRows) {
|
||||
IBlueRow resr = _join_func(lr, rr);
|
||||
if (resr == null)
|
||||
continue;
|
||||
@@ -56,7 +75,7 @@ namespace bts {
|
||||
}
|
||||
}
|
||||
_stopwatch.Stop();
|
||||
Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv?.Count ?? 0} time: {_stopwatch.ElapsedMilliseconds} msecs");
|
||||
Log.Gen.info($"ProximityJoinStream left_count: {leftRows.Count} right_count: {rightRows.Count} result_count: {rv?.Count ?? 0} time: {_stopwatch.ElapsedMilliseconds} msecs");
|
||||
var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
|
||||
myFireEvent(dp);
|
||||
}
|
||||
|
||||
@@ -111,16 +111,21 @@ namespace BCG {
|
||||
_file_address = fileAddress;
|
||||
}
|
||||
|
||||
public static IBlueRowCollection LoadFile(string path) {
|
||||
var sw = Stopwatch.StartNew();
|
||||
bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber opening address{path}");
|
||||
var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(path);
|
||||
sw.Stop();
|
||||
bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {sw.ElapsedMilliseconds} msecs");
|
||||
return rows;
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
//_stopwatch.Start();
|
||||
//DataFrame df = DataFrame.LoadCsv(_file_address);
|
||||
//_stopwatch.Stop();
|
||||
//bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
|
||||
_stopwatch.Start();
|
||||
bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber opening address{_file_address}");
|
||||
IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address);
|
||||
_stopwatch.Stop();
|
||||
bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
|
||||
var rows = LoadFile(_file_address);
|
||||
DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
|
||||
myFireEvent(dp_out);
|
||||
}
|
||||
|
||||
@@ -36,24 +36,6 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public void loadPacket(DataPacket dp) { myFireEvent(dp); }
|
||||
|
||||
/// <summary>
|
||||
/// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc
|
||||
/// The other arguments are provided by the user.
|
||||
/// </summary>
|
||||
/// <param name="rows"></param>
|
||||
/// <param name="streamType"></param>
|
||||
/// <param name="args"></param>
|
||||
/// <returns></returns>
|
||||
public static IBlueRowCollection statelessOp(IBlueRowCollection rows, Type streamType, params object[] args) {
|
||||
ArrayList totArgs = new ArrayList(args);
|
||||
BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream();
|
||||
totArgs.Insert(0, v1);
|
||||
ViewStream retVal = Activator.CreateInstance(streamType, totArgs.ToArray()) as ViewStream;
|
||||
v1.loadCollection(rows);
|
||||
return retVal.rows;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -4,11 +4,16 @@ using System.IO;
|
||||
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using bts;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
[DataStreamUserRespresentation("textWriter")]
|
||||
public class TextWriterStream : ViewStream, IWorker
|
||||
{
|
||||
IBlueRowCollection? rows;
|
||||
#region constructor
|
||||
public TextWriterStream(ViewStream stream, string fileNamePrefix, int writeFreqSecs)
|
||||
: this(ViewStreamDatabase.instance, stream, fileNamePrefix, writeFreqSecs)
|
||||
@@ -53,13 +58,14 @@ namespace BCG
|
||||
{
|
||||
myGotUpdate = false;
|
||||
myNextWrite = DateTime.Now.AddSeconds(myWriteFreqSecs);
|
||||
myWrite(myStream.rows);
|
||||
myWrite(Log.notnull(rows));
|
||||
}
|
||||
}
|
||||
#endregion
|
||||
|
||||
private void myStreamChange(object o)
|
||||
{
|
||||
rows = ExceptionAssert.Argument.Is<IDataPacket>(o).newRows;
|
||||
myGotUpdate = true;
|
||||
}
|
||||
|
||||
|
||||
@@ -32,37 +32,6 @@ namespace BCG
|
||||
?? (myRealSource as ViewStream)?.StreamName
|
||||
?? GetType().Name;
|
||||
|
||||
public IBlueRowCollection rows
|
||||
{
|
||||
get
|
||||
{
|
||||
if (myRealSource != null)
|
||||
{
|
||||
// BlueEventLog.logAssert (myRealSource is ViewStream);
|
||||
|
||||
return (myRealSource as ViewStream).rows;
|
||||
}
|
||||
else
|
||||
{
|
||||
// used to force a computation of a stream which
|
||||
// happens to have no subscribers
|
||||
onChange += new DataStream.ChangeDelegate(myDummy);
|
||||
try
|
||||
{
|
||||
|
||||
if (myLastValue != null)
|
||||
return ((DataPacket) myLastValue).newRows;
|
||||
else
|
||||
return null;
|
||||
}
|
||||
finally
|
||||
{
|
||||
onChange -= new DataStream.ChangeDelegate(myDummy);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected override object myGetValueForInitialSubscription()
|
||||
{
|
||||
if (myLastValue == null)
|
||||
|
||||
+3
-2
@@ -63,8 +63,9 @@
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
var file_path = Path.Combine(SQLiteHelper.location, $"mbp1_{_fcc.symbol}_small.csv".ToLower());
|
||||
var csv_rd = new CSVReaderStream(file_path, false);
|
||||
_rows.rows = csv_rd.rows.to_list().OrderBy(r => r.get<string>("ts_event")).ToList();
|
||||
var rows = BCG.CSVReaderStream.LoadFile(file_path);
|
||||
_rows.rows = new(rows.Count);
|
||||
_rows.rows.AddRange(rows.OfType<IBlueRow>().OrderBy(r => r.get<string>("ts_event")));
|
||||
_rows.pointer = 0;
|
||||
_vsd.addWorker(this);
|
||||
}
|
||||
|
||||
+3
-1
@@ -63,7 +63,9 @@
|
||||
protected override void myOnFirstSubscriber() {
|
||||
var file_path = Path.Combine(SQLiteHelper.location, $"trades_{_fcc.symbol}.csv");
|
||||
var csv_rd = new CSVReaderStream(file_path, false);
|
||||
_rows.rows = csv_rd.rows.to_list().OrderBy(r => r.get<string>("ts_event")).ToList();
|
||||
var rows = BCG.CSVReaderStream.LoadFile(file_path);
|
||||
_rows.rows = new(rows.Count);
|
||||
_rows.rows.AddRange(rows.OfType<IBlueRow>().OrderBy(r => r.get<string>("ts_event")));
|
||||
_rows.pointer = 0;
|
||||
_vsd.addWorker(this);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user