TimeSeries and TimeBucketStreams, Stream offloading etc

This commit is contained in:
Taha committed 2023-10-23 14:18:31 -04:00
1 parent 33030b1212
commit 88bf04e5f1
31 files changed
+1823 -836

No files matched your search

@@ -0,0 +1,24 @@
using BCG;
namespace BCG
{
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
[DataStreamUserRespresentation("add_to_cloud")]
public class AddToCloudStream : ViewStream
{
private readonly string _name, _stream_exp;
private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
public AddToCloudStream(string name, string stream_exp)
: this(name, stream_exp, 0)
{
}
public AddToCloudStream(string name, string stream_exp, int persist)
{
_name = name;
_stream_exp = stream_exp;
_requestor.add_to_cloud(_name, _stream_exp, persist == 1);
mySetRealSource(new TableStream("cloud_streams"));
}
}
}
@@ -0,0 +1,100 @@
using BCG;
using System;
using System.Collections.Generic;
using mbts.config;
using bts.utils;
using System.Threading.Tasks;
using Bts.DataStreamExpressions;
using System.Collections.Generic;
using BCG.Utilities.General;
namespace BCG
{
// listens to the requests posted on "cloud_streams" request table, and
// if unfulfilled starts a ViewPubsliher
public class CloudStreamManager : IWorker
{
public class StreamPubUnit
{
public string name;
public string stream_expression;
public ViewStream stream;
public bool persist;
public ViewPublisher view_publisher;
public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
{
name = name_;
stream_expression = stream_expression_;
stream = stream_;
persist = persist_;
view_publisher = view_publisher_;
}
}
static ViewStream build_stream(string stream_expression)
{
try
{
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv;
}
catch (Exception ex)
{
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
return null;
}
}
private IDictionary<string, StreamPubUnit>
_stream_pub_map = new Dictionary<string, StreamPubUnit>();
private ViewStream _stream = new TableStream("cloud_streams");
private PubTableBroker _broker = new PubTableBroker();
public CloudStreamManager()
{
_stream.onChange += _on_str;
}
private void _on_str(object obj)
{
DataPacket dp = obj as DataPacket;
foreach (IBlueRow r in dp.newRows)
{
string name = r.get<string>("name");
bool persist = r.get<bool>("persist");
string stream_exp = r.get<string>("stream_exp");
if (!_stream_pub_map.ContainsKey(name))
{
_assign(name, stream_exp, persist);
}
}
}
private void _assign(string name, string stream_expression, bool persist)
{
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
ViewStream stream = build_stream(stream_expression);
_expose_to_ws(name, stream);
if (persist)
{
stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs
}
ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
_stream_pub_map.Add(name, u);
}
private void _expose_to_ws(string name, ViewStream stream)
{
// TODO: john
}
public void doWork()
{
}
}
}
@@ -0,0 +1,59 @@
using BCG;
using System.Linq;
namespace BCG
{
public class StreamOffloadRequestor
{
public class OffloadOp
{
string name;
string stream_exp;
}
private IBlueTableSchema _schema;
private IBlueRowCollection _rows;
private PubTableBroker _broker = new PubTableBroker();
ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
public StreamOffloadRequestor()
{
_schema = new BlueTableSchema("cloud_streams",
new ColumnInfo("id", 0, typeof(int)),
new ColumnInfo("name", 1, typeof(string)),
new ColumnInfo("persist", 2, typeof(bool)),
new ColumnInfo("stream_exp", 3, typeof(string)));
_existing_offloads_vs.onChange += _on_str;
}
private void _on_str(object obj)
{
DataPacket dp = obj as DataPacket;
_rows = dp.newRows;
// this is needed so the broker knows the state of the table
_broker.createTable(_rows);
}
public void add_to_cloud(string name, string stream_exp, bool persist)
{
// check if it exists
if (_rows == null)
{
BlueTable table = new BlueTable(_schema);
IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
table.addRow(nr);
_broker.createTable(table);
// create table
}
else
{
if (exists(name))
return;
IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
_broker.addRow(nr);
}
}
bool exists(string name)
{
return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
}
}
}