TimeSeries and TimeBucketStreams, Stream offloading etc
This commit is contained in:
1 parent
33030b1212
commit
88bf04e5f1
31 files changed
+1823
-836
No files matched your search
@@ -0,0 +1,24 @@
|
||||
using BCG;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
|
||||
[DataStreamUserRespresentation("add_to_cloud")]
|
||||
public class AddToCloudStream : ViewStream
|
||||
{
|
||||
private readonly string _name, _stream_exp;
|
||||
private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
|
||||
public AddToCloudStream(string name, string stream_exp)
|
||||
: this(name, stream_exp, 0)
|
||||
{
|
||||
}
|
||||
|
||||
public AddToCloudStream(string name, string stream_exp, int persist)
|
||||
{
|
||||
_name = name;
|
||||
_stream_exp = stream_exp;
|
||||
_requestor.add_to_cloud(_name, _stream_exp, persist == 1);
|
||||
mySetRealSource(new TableStream("cloud_streams"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
using mbts.config;
|
||||
using bts.utils;
|
||||
using System.Threading.Tasks;
|
||||
using Bts.DataStreamExpressions;
|
||||
using System.Collections.Generic;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
// listens to the requests posted on "cloud_streams" request table, and
|
||||
// if unfulfilled starts a ViewPubsliher
|
||||
public class CloudStreamManager : IWorker
|
||||
{
|
||||
public class StreamPubUnit
|
||||
{
|
||||
public string name;
|
||||
public string stream_expression;
|
||||
public ViewStream stream;
|
||||
public bool persist;
|
||||
public ViewPublisher view_publisher;
|
||||
|
||||
public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
|
||||
{
|
||||
name = name_;
|
||||
stream_expression = stream_expression_;
|
||||
stream = stream_;
|
||||
persist = persist_;
|
||||
view_publisher = view_publisher_;
|
||||
}
|
||||
}
|
||||
static ViewStream build_stream(string stream_expression)
|
||||
{
|
||||
try
|
||||
{
|
||||
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
|
||||
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||
return rv;
|
||||
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
private IDictionary<string, StreamPubUnit>
|
||||
_stream_pub_map = new Dictionary<string, StreamPubUnit>();
|
||||
private ViewStream _stream = new TableStream("cloud_streams");
|
||||
private PubTableBroker _broker = new PubTableBroker();
|
||||
|
||||
public CloudStreamManager()
|
||||
{
|
||||
_stream.onChange += _on_str;
|
||||
|
||||
}
|
||||
private void _on_str(object obj)
|
||||
{
|
||||
DataPacket dp = obj as DataPacket;
|
||||
foreach (IBlueRow r in dp.newRows)
|
||||
{
|
||||
string name = r.get<string>("name");
|
||||
bool persist = r.get<bool>("persist");
|
||||
string stream_exp = r.get<string>("stream_exp");
|
||||
if (!_stream_pub_map.ContainsKey(name))
|
||||
{
|
||||
_assign(name, stream_exp, persist);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void _assign(string name, string stream_expression, bool persist)
|
||||
{
|
||||
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
|
||||
ViewStream stream = build_stream(stream_expression);
|
||||
_expose_to_ws(name, stream);
|
||||
if (persist)
|
||||
{
|
||||
stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs
|
||||
}
|
||||
ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
|
||||
StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
|
||||
_stream_pub_map.Add(name, u);
|
||||
}
|
||||
|
||||
private void _expose_to_ws(string name, ViewStream stream)
|
||||
{
|
||||
// TODO: john
|
||||
}
|
||||
|
||||
public void doWork()
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
using BCG;
|
||||
|
||||
using System.Linq;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class StreamOffloadRequestor
|
||||
{
|
||||
public class OffloadOp
|
||||
{
|
||||
string name;
|
||||
string stream_exp;
|
||||
}
|
||||
private IBlueTableSchema _schema;
|
||||
private IBlueRowCollection _rows;
|
||||
private PubTableBroker _broker = new PubTableBroker();
|
||||
ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
|
||||
public StreamOffloadRequestor()
|
||||
{
|
||||
_schema = new BlueTableSchema("cloud_streams",
|
||||
new ColumnInfo("id", 0, typeof(int)),
|
||||
new ColumnInfo("name", 1, typeof(string)),
|
||||
new ColumnInfo("persist", 2, typeof(bool)),
|
||||
new ColumnInfo("stream_exp", 3, typeof(string)));
|
||||
_existing_offloads_vs.onChange += _on_str;
|
||||
}
|
||||
private void _on_str(object obj)
|
||||
{
|
||||
DataPacket dp = obj as DataPacket;
|
||||
_rows = dp.newRows;
|
||||
// this is needed so the broker knows the state of the table
|
||||
_broker.createTable(_rows);
|
||||
}
|
||||
public void add_to_cloud(string name, string stream_exp, bool persist)
|
||||
{
|
||||
// check if it exists
|
||||
if (_rows == null)
|
||||
{
|
||||
BlueTable table = new BlueTable(_schema);
|
||||
IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
|
||||
table.addRow(nr);
|
||||
_broker.createTable(table);
|
||||
// create table
|
||||
}
|
||||
else
|
||||
{
|
||||
if (exists(name))
|
||||
return;
|
||||
IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
|
||||
_broker.addRow(nr);
|
||||
}
|
||||
}
|
||||
|
||||
bool exists(string name)
|
||||
{
|
||||
return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user