TimeSeries and TimeBucketStreams, Stream offloading etc

This commit is contained in:
Taha committed 2023-10-23 14:18:31 -04:00
1 parent 33030b1212
commit 88bf04e5f1
31 files changed
+1823 -836

No files matched your search

@@ -0,0 +1,24 @@
using BCG;
namespace BCG
{
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
[DataStreamUserRespresentation("add_to_cloud")]
public class AddToCloudStream : ViewStream
{
private readonly string _name, _stream_exp;
private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
public AddToCloudStream(string name, string stream_exp)
: this(name, stream_exp, 0)
{
}
public AddToCloudStream(string name, string stream_exp, int persist)
{
_name = name;
_stream_exp = stream_exp;
_requestor.add_to_cloud(_name, _stream_exp, persist == 1);
mySetRealSource(new TableStream("cloud_streams"));
}
}
}
@@ -0,0 +1,100 @@
using BCG;
using System;
using System.Collections.Generic;
using mbts.config;
using bts.utils;
using System.Threading.Tasks;
using Bts.DataStreamExpressions;
using System.Collections.Generic;
using BCG.Utilities.General;
namespace BCG
{
// listens to the requests posted on "cloud_streams" request table, and
// if unfulfilled starts a ViewPubsliher
public class CloudStreamManager : IWorker
{
public class StreamPubUnit
{
public string name;
public string stream_expression;
public ViewStream stream;
public bool persist;
public ViewPublisher view_publisher;
public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
{
name = name_;
stream_expression = stream_expression_;
stream = stream_;
persist = persist_;
view_publisher = view_publisher_;
}
}
static ViewStream build_stream(string stream_expression)
{
try
{
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv;
}
catch (Exception ex)
{
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
return null;
}
}
private IDictionary<string, StreamPubUnit>
_stream_pub_map = new Dictionary<string, StreamPubUnit>();
private ViewStream _stream = new TableStream("cloud_streams");
private PubTableBroker _broker = new PubTableBroker();
public CloudStreamManager()
{
_stream.onChange += _on_str;
}
private void _on_str(object obj)
{
DataPacket dp = obj as DataPacket;
foreach (IBlueRow r in dp.newRows)
{
string name = r.get<string>("name");
bool persist = r.get<bool>("persist");
string stream_exp = r.get<string>("stream_exp");
if (!_stream_pub_map.ContainsKey(name))
{
_assign(name, stream_exp, persist);
}
}
}
private void _assign(string name, string stream_expression, bool persist)
{
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
ViewStream stream = build_stream(stream_expression);
_expose_to_ws(name, stream);
if (persist)
{
stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs
}
ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
_stream_pub_map.Add(name, u);
}
private void _expose_to_ws(string name, ViewStream stream)
{
// TODO: john
}
public void doWork()
{
}
}
}
@@ -0,0 +1,59 @@
using BCG;
using System.Linq;
namespace BCG
{
public class StreamOffloadRequestor
{
public class OffloadOp
{
string name;
string stream_exp;
}
private IBlueTableSchema _schema;
private IBlueRowCollection _rows;
private PubTableBroker _broker = new PubTableBroker();
ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
public StreamOffloadRequestor()
{
_schema = new BlueTableSchema("cloud_streams",
new ColumnInfo("id", 0, typeof(int)),
new ColumnInfo("name", 1, typeof(string)),
new ColumnInfo("persist", 2, typeof(bool)),
new ColumnInfo("stream_exp", 3, typeof(string)));
_existing_offloads_vs.onChange += _on_str;
}
private void _on_str(object obj)
{
DataPacket dp = obj as DataPacket;
_rows = dp.newRows;
// this is needed so the broker knows the state of the table
_broker.createTable(_rows);
}
public void add_to_cloud(string name, string stream_exp, bool persist)
{
// check if it exists
if (_rows == null)
{
BlueTable table = new BlueTable(_schema);
IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
table.addRow(nr);
_broker.createTable(table);
// create table
}
else
{
if (exists(name))
return;
IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
_broker.addRow(nr);
}
}
bool exists(string name)
{
return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
}
}
}
@@ -8,6 +8,7 @@ using mbts.spark;
namespace BCG
{
[DataStreamUserRespresentation("fastJoin")]
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
{
private readonly NonIncJoinSpec[] _join_specs;
@@ -23,7 +24,7 @@ namespace BCG
public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
ViewStream right, string right_join_cols, string right_proj)
: this(new Triplet<ViewStream, string, string>[]
: this(new Triplet<ViewStream, string, string>[]
{
new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
@@ -31,7 +32,7 @@ namespace BCG
{ }
public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
: this (NonIncJoinSpec.from_triplet (join_specs), null, 0)
: this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
{ }
public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
@@ -102,8 +103,8 @@ namespace BCG
{
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
d.constribute_cols_to_schema (cis);
_output_schema = new BlueTableSchema ("NonIncJoinStream", cis.ToArray());
d.constribute_cols_to_schema(cis);
_output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
}
}
@@ -24,6 +24,8 @@ namespace BCG {
private void _onstream_change(object o) {
DataPacket dp = o as DataPacket;
BlueDataUtility.validate_table_schema(dp.newRows);
BlueDataUtility.validate_table_schema(dp.addedRows);
if (_last_recv_rows == null) {
_last_recv_rows = dp.newRows;
} else {
@@ -1,37 +1,47 @@
using Google.Protobuf.WellKnownTypes;
using System;
using BCG.Utilities.General;
namespace BCG {
namespace BCG
{
// Keeps adding changing rows to the stream, hence preserving the history
// and creating a time series, for example this can be used to make bbo updates
// into a time series of prices/qty
public class TimeSeriesStream : ViewStream {
public class TimeSeriesStream : ViewStream
{
private BlueTable _table;
private ViewStream _stream;
private int _id = 0;
public TimeSeriesStream(ViewStream stream) {
public TimeSeriesStream(ViewStream stream)
{
_stream = stream;
}
protected override void myOnFirstSubscriber() {
protected override void myOnFirstSubscriber()
{
_stream.onChange += new ChangeDelegate(_onstream_change);
}
protected override void myOnLastSubscriber() {
protected override void myOnLastSubscriber()
{
_stream.onChange -= new ChangeDelegate(_onstream_change);
}
private void _onstream_change(object o) {
private void _onstream_change(object o)
{
DataPacket dp = o as DataPacket;
DateTime now = DateTime.Now;
if (_table == null) {
if (_table == null)
{
IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
_table = new BlueTable(schema);
}
BlueTable added_rows = new BlueTable(_table.schema);
foreach (IBlueRow r in dp.addedRows) {
foreach (IBlueRow r in dp.addedRows)
{
MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
ar.setValue("timestamp", now);
ar.setValue("id", _id++);
@@ -42,4 +52,118 @@ namespace BCG {
}
}
}
public class TimeBucketStream : ViewStream, IStreamMapper
{
private ViewStream _src;
private readonly string _time_stamp_col;
private IBlueTableSchema _schema;
private int _bucket_size_seconds;
public TimeBucketStream(ViewStream input, string time_stamp_col, int bucket_size_seconds)
{
_time_stamp_col = time_stamp_col;
_bucket_size_seconds = bucket_size_seconds;
var vs = new MapStream(input, this);
mySetRealSource(vs);
}
#region IStreamMapper Members
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema)
{
IBlueTableSchema rv = input_schema;
//rv = BlueDataUtility.remove_columns(rv, _time_stamp_col);
rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int));
return rv;
}
#endregion
#region IDynamicStreamMapper Members
public IBlueRow map(IBlueRow row)
{
DateTime ts = row.get<DateTime>(_time_stamp_col);
TimeSpan time_of_day = (ts - ts.Date);
int seconds = Convert.ToInt32(time_of_day.TotalSeconds);
double buckets = seconds / _bucket_size_seconds;
int bucket_number = Convert.ToInt32(Math.Floor(buckets));
MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
rv.setValue("bucket_number", bucket_number);
return rv;
}
#endregion
}
// public class ThrottleStream : ViewStream, IWorker
// {
// private ViewStreamDatabase _vsd;
// private ViewStream _stream;
// private int _throttle_ms;
// private TimeSpan _min_fire_period;
// private DateTime _last_Fire_time = DateTime.MinValue;
// private DataPacket _unfired_dp;
// public ThrottleStream(ViewStreamDatabase vsd, ViewStream base_stream, int throttle_ms)
// {
// _vsd = vsd;
// _stream = base_stream;
// _throttle_ms = throttle_ms;
// _min_fire_period = TimeSpan.FromMilliSeconds(throttle_ms);
// }
// protected override void myOnFirstSubscriber()
// {
// _vsd.addWorker(this);
// _stream.onChange += new ChangeDelegate(_onstream_change);
// }
// protected override void myOnLastSubscriber()
// {
// _vsd.removeWorker(this);
// _stream.onChange -= new ChangeDelegate(_onstream_change);
// }
// private void _onstream_change(object obj)
// {
// DataPacket dp = obj as DataPacket;
// if (DateTime.Now - _last_Fire_time >= _min_fire_period)
// {
// // ok to fire here
// }
// else
// {
// _last_cached_state = dp.newRows;
// }
// _possibly_fire(dp.newRows);
// }
// public void doWork()
// {
// if (_unfired_dp != null)
// {
// }
// }
// private void _possibly_fire()
// {
// if (_unfired_dp != null)
// {
// if (DateTime.Now - _last_fire_time > _min_fire_period)
// {
// DataPacket dp = BlueDataUtility.build_dp_inc(_unfired_dp.newRows, myLastValue);
// _unfired_dp = null;
// _last_Fire_time = DateTime.Now;
// myFireEvent(dp);
// }
// }
// }
// }
}
@@ -6,7 +6,7 @@ using BCG.Utilities;
namespace BCG
{
[DataStreamUserRespresentation("csvSaver")]
[DataStreamUserRespresentation("CSVSaver")]
public class CSVSaverStream : ViewStream, ISideEffectStream
{
private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
Whitespace-only changes.
@@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Collections;
namespace BCG
{
[DataStreamUserRepresentation("timeBucket")]
public class TimeBucketStream : ViewStream, IStreamMapper
{
private ViewStream _src;
private readonly string _time_stamp_col;
private IBlueTableSchema _schema;
private uint _bucket_size_seconds;
public TimeBucketStream(ViewStream input, string time_stamp_col, uint bucket_size_seconds)
{
_time_stamp_col = time_stamp_col;
_bucket_size_seconds = bucket_size_seconds;
}
#region IStreamMapper Members
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema)
{
IBlueTableSchema rv = input_schema;
//rv = BlueDataUtility.remove_columns(rv, _time_stamp_col);
rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int));
return rv;
}
#endregion
#region IDynamicStreamMapper Members
public IBlueRow map(IBlueRow row)
{
DateTime ts = row.get<DateTime>(_time_stamp_col);
TimeSpan time_of_day = (ts - ts.Date)
int seconds = Convert.ToInt32(time_of_day.TotalSeconds);
int buckets = seconds / _bucket_size_seconds;
int bucket_number = Math.Floor(buckets);
MutableBlueRow rv = new MutableBlueRow(row, getOutputSchema(row.schema))
rv.setValue("bucket_number", bucket_number);
}
#endregion
}
}
@@ -1,10 +1,9 @@
using System;
using BCG.Utilities.General;
namespace BCG
{
public abstract class TimeUnitStream : ViewStream, IWorker
{
namespace BCG {
public abstract class TimeUnitStream : ViewStream, IWorker {
protected TimeSpan myOffset;
protected DateTime myConstructionTime;
protected int mySizeOfStream;
@@ -13,87 +12,80 @@ namespace BCG
protected ViewStreamDatabase myDb;
protected int myId = 0;
public TimeUnitStream (DateTime zeroTime, TimeSpan offSet, int sizeOfStream, ViewStreamDatabase db)
{
public TimeUnitStream(DateTime zeroTime, TimeSpan offSet, int sizeOfStream, ViewStreamDatabase db) {
myConstructionTime = DateTime.Now;
myZeroTime = zeroTime;
myOffset = offSet;
mySizeOfStream = sizeOfStream;
myDb = db;
myCollection = new CircularRowCollection (
new BlueTableSchema ("Time",
new ColumnInfo ("id", 0, typeof(Int32)), new ColumnInfo ("time", 1, typeof(DateTime))),
myCollection = new CircularRowCollection(
new BlueTableSchema("Time",
new ColumnInfo("id", 0, typeof(Int32)), new ColumnInfo("time", 1, typeof(DateTime))),
mySizeOfStream);
}
}
#region ViewStream overriden methods
protected override void myOnFirstSubscriber()
{
myDb.addWorker (this);
myInit ();
protected override void myOnFirstSubscriber() {
myDb.addWorker(this);
myInit();
}
protected override void myOnLastSubscriber()
{
myDb.removeWorker (this);
protected override void myOnLastSubscriber() {
myDb.removeWorker(this);
}
#endregion
protected virtual void myInit ()
{
protected virtual void myInit() {
int periods = (int)
(DateTime.Now - myConstructionTime).TotalMilliseconds / (int) myOffset.TotalMilliseconds;
(DateTime.Now - myConstructionTime).TotalMilliseconds / (int)myOffset.TotalMilliseconds;
DateTime dt = myGetLatestRow() == null ? myRoundTime(myZeroTime) :
myRoundTime (myZeroTime + new TimeSpan(myOffset.Ticks * periods));
DateTime dt = myGetLatestRow() == null ? myRoundTime(myZeroTime) :
myRoundTime(myZeroTime + new TimeSpan(myOffset.Ticks * periods));
if (myGetLatestRow() == null ||
!myGetLatestRow()["time"].Equals(dt))
{
IBlueRow ar = new BlueRow (myCollection.schema,
!myGetLatestRow()["time"].Equals(dt)) {
IBlueRow ar = new BlueRow(myCollection.schema,
myId++, dt);
IBlueRow dr = myCollection.addRowWithReturn (ar);
myFireEvent (new DataPacket (myCollection,
myConvertToCollection (ar),
myConvertToCollection (dr)));
IBlueRow dr = myCollection.addRowWithReturn(ar);
if (n_updates <= 3000)
myFireEvent(new DataPacket(myCollection,
myConvertToCollection(ar),
myConvertToCollection(dr)));
}
}
#region IWorker methods
public void doWork ()
{
myInit ();
public void doWork() {
myInit();
}
#endregion
protected abstract DateTime myRoundTime (DateTime dt);
protected abstract DateTime myRoundTime(DateTime dt);
#region Private Helper methods
private IBlueRow myGetLatestRow ()
{
private IBlueRow myGetLatestRow() {
return myCollection.Count == 0 ?
null :
BlueDataUtility.extractRow (myCollection, myCollection.Count - 1);
BlueDataUtility.extractRow(myCollection, myCollection.Count - 1);
}
private IBlueRowCollection myConvertToCollection (IBlueRow r)
{
BlueTable retVal = new BlueTable (myCollection.schema);
private IBlueRowCollection myConvertToCollection(IBlueRow r) {
BlueTable retVal = new BlueTable(myCollection.schema);
if (r != null)
retVal.addRow (r);
retVal.addRow(r);
return retVal;
}
#endregion
}
}
}
@@ -1,6 +1,8 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq.Expressions;
using System.Xml.Serialization;
using BCG.Utilities;
@@ -53,7 +55,12 @@ namespace BCG {
}
public virtual void createTable(IBlueRowCollection newTable) {
bts.Log.Gen.info($"PubTableBroker.createTable table: {newTable.schema.tableName}");
string log_msg = $"PubTableBroker.createTable table: {newTable.schema.tableName}\n";
foreach (IBlueRow r in newTable) {
log_msg += $"id: {r["id"]}";
}
bts.Log.Gen.info(log_msg);
_check_schema_consistency(newTable.schema);
myCheckTransaction();
IPubTablePublisher table_publisher = null;
var name_key = newTable.schema.tableName.ToLower();
@@ -71,6 +78,9 @@ namespace BCG {
/// </summary>
/// <param name="r"></param>
public virtual void addRow(IBlueRow r) {
string log_msg = $"PubTableBroker.addRow table: {r.schema.tableName} id: {r["id"]}";
bts.Log.Gen.info(log_msg);
_check_schema_consistency(r.schema);
myCheckTransaction();
IPubTablePublisher table_publisher = null;
var name_key = r.schema.tableName.ToLower();
@@ -86,6 +96,9 @@ namespace BCG {
/// </summary>
/// <param name="r"></param>
public virtual void deleteRow(IBlueRow r) {
string log_msg = $"PubTableBroker.deleteRow table: {r.schema.tableName} id: {r["id"]}";
bts.Log.Gen.info(log_msg);
_check_schema_consistency(r.schema);
myCheckTransaction();
IPubTablePublisher table_publisher = null;
var name_key = r.schema.tableName.ToLower();
@@ -101,6 +114,9 @@ namespace BCG {
/// </summary>
/// <param name="r"></param>
public virtual void changeRow(IBlueRow r) {
string log_msg = $"PubTableBroker.changeRow table: {r.schema.tableName} id: {r["id"]}";
bts.Log.Gen.info(log_msg);
_check_schema_consistency(r.schema);
myCheckTransaction();
IPubTablePublisher table_publisher = null;
var name_key = r.schema.tableName.ToLower();
@@ -118,7 +134,16 @@ namespace BCG {
public virtual void makeBatchChanges(IBlueRowCollection newRows,
IBlueRowCollection deletedRows,
IBlueRowCollection addedRows) {
myCheckTransaction();
string log_msg = $"PubTableBroker.makeBatchChanges {deletedRows.Count} {addedRows.Count} table: {newRows.schema.tableName} deleted_rows: ";
foreach (IBlueRow r in deletedRows)
log_msg += $"id: {r["id"]} ";
log_msg += "addedRows: ";
foreach (IBlueRow r in addedRows)
log_msg += $"id: {r["id"]}";
bts.Log.Gen.info(log_msg);
_check_schema_consistency(newRows.schema);
_check_schema_consistency(deletedRows.schema);
_check_schema_consistency(addedRows.schema);
IPubTablePublisher table_publisher = null;
var name_key = newRows.schema.tableName.ToLower();
if (!_publisher_map.TryGetValue(name_key, out table_publisher))
@@ -144,8 +169,16 @@ namespace BCG {
}
protected virtual IPubTablePublisher _create_pubtable_publisher(string tag) {
bts.Log.Gen.info($"PubTableBroker._create_pubtable_publisher tag: {tag}");
return new PubTablePublisher(tag, _publisher_core);
}
private void _check_schema_consistency(IBlueTableSchema schema_arg) {
IBlueTableSchema schema;
if (_schema_map.TryGetValue(schema_arg.tableName, out schema)) {
bts.Log.assert(BlueDataUtility.compareSchemas(schema, schema_arg));
}
}
}
}
@@ -14,7 +14,7 @@ namespace BCG {
/// is not uniquely chosen)
public ViewPublisher(string name, PubTableBroker br, ViewStream vs) {
myName = name;
myView = vs;//String.IsNullOrEmpty(name) ? vs : new ProjectStream(vs, "!", myName);
myView = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
myBroker = br;
myFirstTime = true;
myView.onChange += new DataStream.ChangeDelegate(myChangeHandler);