TimeSeries and TimeBucketStreams, Stream offloading etc
This commit is contained in:
1 parent
33030b1212
commit
88bf04e5f1
31 files changed
+1823
-836
No files matched your search
@@ -0,0 +1,24 @@
|
||||
using BCG;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
|
||||
[DataStreamUserRespresentation("add_to_cloud")]
|
||||
public class AddToCloudStream : ViewStream
|
||||
{
|
||||
private readonly string _name, _stream_exp;
|
||||
private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
|
||||
public AddToCloudStream(string name, string stream_exp)
|
||||
: this(name, stream_exp, 0)
|
||||
{
|
||||
}
|
||||
|
||||
public AddToCloudStream(string name, string stream_exp, int persist)
|
||||
{
|
||||
_name = name;
|
||||
_stream_exp = stream_exp;
|
||||
_requestor.add_to_cloud(_name, _stream_exp, persist == 1);
|
||||
mySetRealSource(new TableStream("cloud_streams"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
using mbts.config;
|
||||
using bts.utils;
|
||||
using System.Threading.Tasks;
|
||||
using Bts.DataStreamExpressions;
|
||||
using System.Collections.Generic;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
// listens to the requests posted on "cloud_streams" request table, and
|
||||
// if unfulfilled starts a ViewPubsliher
|
||||
public class CloudStreamManager : IWorker
|
||||
{
|
||||
public class StreamPubUnit
|
||||
{
|
||||
public string name;
|
||||
public string stream_expression;
|
||||
public ViewStream stream;
|
||||
public bool persist;
|
||||
public ViewPublisher view_publisher;
|
||||
|
||||
public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
|
||||
{
|
||||
name = name_;
|
||||
stream_expression = stream_expression_;
|
||||
stream = stream_;
|
||||
persist = persist_;
|
||||
view_publisher = view_publisher_;
|
||||
}
|
||||
}
|
||||
static ViewStream build_stream(string stream_expression)
|
||||
{
|
||||
try
|
||||
{
|
||||
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
|
||||
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||
return rv;
|
||||
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
private IDictionary<string, StreamPubUnit>
|
||||
_stream_pub_map = new Dictionary<string, StreamPubUnit>();
|
||||
private ViewStream _stream = new TableStream("cloud_streams");
|
||||
private PubTableBroker _broker = new PubTableBroker();
|
||||
|
||||
public CloudStreamManager()
|
||||
{
|
||||
_stream.onChange += _on_str;
|
||||
|
||||
}
|
||||
private void _on_str(object obj)
|
||||
{
|
||||
DataPacket dp = obj as DataPacket;
|
||||
foreach (IBlueRow r in dp.newRows)
|
||||
{
|
||||
string name = r.get<string>("name");
|
||||
bool persist = r.get<bool>("persist");
|
||||
string stream_exp = r.get<string>("stream_exp");
|
||||
if (!_stream_pub_map.ContainsKey(name))
|
||||
{
|
||||
_assign(name, stream_exp, persist);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void _assign(string name, string stream_expression, bool persist)
|
||||
{
|
||||
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
|
||||
ViewStream stream = build_stream(stream_expression);
|
||||
_expose_to_ws(name, stream);
|
||||
if (persist)
|
||||
{
|
||||
stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs
|
||||
}
|
||||
ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
|
||||
StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
|
||||
_stream_pub_map.Add(name, u);
|
||||
}
|
||||
|
||||
private void _expose_to_ws(string name, ViewStream stream)
|
||||
{
|
||||
// TODO: john
|
||||
}
|
||||
|
||||
public void doWork()
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
using BCG;
|
||||
|
||||
using System.Linq;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class StreamOffloadRequestor
|
||||
{
|
||||
public class OffloadOp
|
||||
{
|
||||
string name;
|
||||
string stream_exp;
|
||||
}
|
||||
private IBlueTableSchema _schema;
|
||||
private IBlueRowCollection _rows;
|
||||
private PubTableBroker _broker = new PubTableBroker();
|
||||
ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
|
||||
public StreamOffloadRequestor()
|
||||
{
|
||||
_schema = new BlueTableSchema("cloud_streams",
|
||||
new ColumnInfo("id", 0, typeof(int)),
|
||||
new ColumnInfo("name", 1, typeof(string)),
|
||||
new ColumnInfo("persist", 2, typeof(bool)),
|
||||
new ColumnInfo("stream_exp", 3, typeof(string)));
|
||||
_existing_offloads_vs.onChange += _on_str;
|
||||
}
|
||||
private void _on_str(object obj)
|
||||
{
|
||||
DataPacket dp = obj as DataPacket;
|
||||
_rows = dp.newRows;
|
||||
// this is needed so the broker knows the state of the table
|
||||
_broker.createTable(_rows);
|
||||
}
|
||||
public void add_to_cloud(string name, string stream_exp, bool persist)
|
||||
{
|
||||
// check if it exists
|
||||
if (_rows == null)
|
||||
{
|
||||
BlueTable table = new BlueTable(_schema);
|
||||
IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
|
||||
table.addRow(nr);
|
||||
_broker.createTable(table);
|
||||
// create table
|
||||
}
|
||||
else
|
||||
{
|
||||
if (exists(name))
|
||||
return;
|
||||
IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
|
||||
_broker.addRow(nr);
|
||||
}
|
||||
}
|
||||
|
||||
bool exists(string name)
|
||||
{
|
||||
return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ using mbts.spark;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
[DataStreamUserRespresentation("fastJoin")]
|
||||
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
|
||||
{
|
||||
private readonly NonIncJoinSpec[] _join_specs;
|
||||
@@ -23,7 +24,7 @@ namespace BCG
|
||||
|
||||
public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
|
||||
ViewStream right, string right_join_cols, string right_proj)
|
||||
: this(new Triplet<ViewStream, string, string>[]
|
||||
: this(new Triplet<ViewStream, string, string>[]
|
||||
{
|
||||
new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
|
||||
new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
|
||||
@@ -31,7 +32,7 @@ namespace BCG
|
||||
{ }
|
||||
|
||||
public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
|
||||
: this (NonIncJoinSpec.from_triplet (join_specs), null, 0)
|
||||
: this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
|
||||
{ }
|
||||
|
||||
public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
|
||||
@@ -102,8 +103,8 @@ namespace BCG
|
||||
{
|
||||
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
|
||||
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
||||
d.constribute_cols_to_schema (cis);
|
||||
_output_schema = new BlueTableSchema ("NonIncJoinStream", cis.ToArray());
|
||||
d.constribute_cols_to_schema(cis);
|
||||
_output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,8 @@ namespace BCG {
|
||||
|
||||
private void _onstream_change(object o) {
|
||||
DataPacket dp = o as DataPacket;
|
||||
BlueDataUtility.validate_table_schema(dp.newRows);
|
||||
BlueDataUtility.validate_table_schema(dp.addedRows);
|
||||
if (_last_recv_rows == null) {
|
||||
_last_recv_rows = dp.newRows;
|
||||
} else {
|
||||
|
||||
@@ -1,37 +1,47 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
|
||||
using System;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
namespace BCG {
|
||||
namespace BCG
|
||||
{
|
||||
// Keeps adding changing rows to the stream, hence preserving the history
|
||||
// and creating a time series, for example this can be used to make bbo updates
|
||||
// into a time series of prices/qty
|
||||
public class TimeSeriesStream : ViewStream {
|
||||
public class TimeSeriesStream : ViewStream
|
||||
{
|
||||
private BlueTable _table;
|
||||
private ViewStream _stream;
|
||||
private int _id = 0;
|
||||
|
||||
public TimeSeriesStream(ViewStream stream) {
|
||||
public TimeSeriesStream(ViewStream stream)
|
||||
{
|
||||
_stream = stream;
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
_stream.onChange += new ChangeDelegate(_onstream_change);
|
||||
}
|
||||
protected override void myOnLastSubscriber() {
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
_stream.onChange -= new ChangeDelegate(_onstream_change);
|
||||
}
|
||||
|
||||
private void _onstream_change(object o) {
|
||||
|
||||
private void _onstream_change(object o)
|
||||
{
|
||||
DataPacket dp = o as DataPacket;
|
||||
DateTime now = DateTime.Now;
|
||||
if (_table == null) {
|
||||
if (_table == null)
|
||||
{
|
||||
IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
|
||||
schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
|
||||
_table = new BlueTable(schema);
|
||||
}
|
||||
BlueTable added_rows = new BlueTable(_table.schema);
|
||||
foreach (IBlueRow r in dp.addedRows) {
|
||||
foreach (IBlueRow r in dp.addedRows)
|
||||
{
|
||||
MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
|
||||
ar.setValue("timestamp", now);
|
||||
ar.setValue("id", _id++);
|
||||
@@ -42,4 +52,118 @@ namespace BCG {
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class TimeBucketStream : ViewStream, IStreamMapper
|
||||
{
|
||||
private ViewStream _src;
|
||||
private readonly string _time_stamp_col;
|
||||
private IBlueTableSchema _schema;
|
||||
private int _bucket_size_seconds;
|
||||
|
||||
public TimeBucketStream(ViewStream input, string time_stamp_col, int bucket_size_seconds)
|
||||
{
|
||||
_time_stamp_col = time_stamp_col;
|
||||
_bucket_size_seconds = bucket_size_seconds;
|
||||
var vs = new MapStream(input, this);
|
||||
mySetRealSource(vs);
|
||||
}
|
||||
|
||||
#region IStreamMapper Members
|
||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema)
|
||||
{
|
||||
IBlueTableSchema rv = input_schema;
|
||||
//rv = BlueDataUtility.remove_columns(rv, _time_stamp_col);
|
||||
rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int));
|
||||
return rv;
|
||||
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region IDynamicStreamMapper Members
|
||||
|
||||
public IBlueRow map(IBlueRow row)
|
||||
{
|
||||
DateTime ts = row.get<DateTime>(_time_stamp_col);
|
||||
TimeSpan time_of_day = (ts - ts.Date);
|
||||
int seconds = Convert.ToInt32(time_of_day.TotalSeconds);
|
||||
double buckets = seconds / _bucket_size_seconds;
|
||||
int bucket_number = Convert.ToInt32(Math.Floor(buckets));
|
||||
MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
|
||||
rv.setValue("bucket_number", bucket_number);
|
||||
return rv;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
// public class ThrottleStream : ViewStream, IWorker
|
||||
// {
|
||||
// private ViewStreamDatabase _vsd;
|
||||
// private ViewStream _stream;
|
||||
// private int _throttle_ms;
|
||||
// private TimeSpan _min_fire_period;
|
||||
// private DateTime _last_Fire_time = DateTime.MinValue;
|
||||
// private DataPacket _unfired_dp;
|
||||
// public ThrottleStream(ViewStreamDatabase vsd, ViewStream base_stream, int throttle_ms)
|
||||
// {
|
||||
// _vsd = vsd;
|
||||
// _stream = base_stream;
|
||||
// _throttle_ms = throttle_ms;
|
||||
// _min_fire_period = TimeSpan.FromMilliSeconds(throttle_ms);
|
||||
// }
|
||||
|
||||
// protected override void myOnFirstSubscriber()
|
||||
// {
|
||||
// _vsd.addWorker(this);
|
||||
// _stream.onChange += new ChangeDelegate(_onstream_change);
|
||||
// }
|
||||
// protected override void myOnLastSubscriber()
|
||||
// {
|
||||
// _vsd.removeWorker(this);
|
||||
// _stream.onChange -= new ChangeDelegate(_onstream_change);
|
||||
// }
|
||||
|
||||
// private void _onstream_change(object obj)
|
||||
// {
|
||||
// DataPacket dp = obj as DataPacket;
|
||||
// if (DateTime.Now - _last_Fire_time >= _min_fire_period)
|
||||
// {
|
||||
// // ok to fire here
|
||||
// }
|
||||
// else
|
||||
// {
|
||||
// _last_cached_state = dp.newRows;
|
||||
// }
|
||||
// _possibly_fire(dp.newRows);
|
||||
// }
|
||||
|
||||
// public void doWork()
|
||||
// {
|
||||
// if (_unfired_dp != null)
|
||||
// {
|
||||
|
||||
// }
|
||||
// }
|
||||
|
||||
// private void _possibly_fire()
|
||||
// {
|
||||
// if (_unfired_dp != null)
|
||||
// {
|
||||
// if (DateTime.Now - _last_fire_time > _min_fire_period)
|
||||
// {
|
||||
// DataPacket dp = BlueDataUtility.build_dp_inc(_unfired_dp.newRows, myLastValue);
|
||||
// _unfired_dp = null;
|
||||
// _last_Fire_time = DateTime.Now;
|
||||
// myFireEvent(dp);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// }
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ using BCG.Utilities;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
[DataStreamUserRespresentation("csvSaver")]
|
||||
[DataStreamUserRespresentation("CSVSaver")]
|
||||
public class CSVSaverStream : ViewStream, ISideEffectStream
|
||||
{
|
||||
private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
|
||||
|
||||
Whitespace-only changes.
@@ -0,0 +1,51 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using System.Collections;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
[DataStreamUserRepresentation("timeBucket")]
|
||||
public class TimeBucketStream : ViewStream, IStreamMapper
|
||||
{
|
||||
private ViewStream _src;
|
||||
private readonly string _time_stamp_col;
|
||||
private IBlueTableSchema _schema;
|
||||
private uint _bucket_size_seconds;
|
||||
|
||||
public TimeBucketStream(ViewStream input, string time_stamp_col, uint bucket_size_seconds)
|
||||
{
|
||||
_time_stamp_col = time_stamp_col;
|
||||
_bucket_size_seconds = bucket_size_seconds;
|
||||
}
|
||||
|
||||
#region IStreamMapper Members
|
||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema)
|
||||
{
|
||||
IBlueTableSchema rv = input_schema;
|
||||
//rv = BlueDataUtility.remove_columns(rv, _time_stamp_col);
|
||||
rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int));
|
||||
return rv;
|
||||
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region IDynamicStreamMapper Members
|
||||
|
||||
public IBlueRow map(IBlueRow row)
|
||||
{
|
||||
DateTime ts = row.get<DateTime>(_time_stamp_col);
|
||||
TimeSpan time_of_day = (ts - ts.Date)
|
||||
int seconds = Convert.ToInt32(time_of_day.TotalSeconds);
|
||||
int buckets = seconds / _bucket_size_seconds;
|
||||
int bucket_number = Math.Floor(buckets);
|
||||
MutableBlueRow rv = new MutableBlueRow(row, getOutputSchema(row.schema))
|
||||
rv.setValue("bucket_number", bucket_number);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,9 @@
|
||||
using System;
|
||||
|
||||
using BCG.Utilities.General;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public abstract class TimeUnitStream : ViewStream, IWorker
|
||||
{
|
||||
namespace BCG {
|
||||
public abstract class TimeUnitStream : ViewStream, IWorker {
|
||||
protected TimeSpan myOffset;
|
||||
protected DateTime myConstructionTime;
|
||||
protected int mySizeOfStream;
|
||||
@@ -13,87 +12,80 @@ namespace BCG
|
||||
protected ViewStreamDatabase myDb;
|
||||
protected int myId = 0;
|
||||
|
||||
public TimeUnitStream (DateTime zeroTime, TimeSpan offSet, int sizeOfStream, ViewStreamDatabase db)
|
||||
{
|
||||
public TimeUnitStream(DateTime zeroTime, TimeSpan offSet, int sizeOfStream, ViewStreamDatabase db) {
|
||||
myConstructionTime = DateTime.Now;
|
||||
myZeroTime = zeroTime;
|
||||
myOffset = offSet;
|
||||
mySizeOfStream = sizeOfStream;
|
||||
myDb = db;
|
||||
myCollection = new CircularRowCollection (
|
||||
new BlueTableSchema ("Time",
|
||||
new ColumnInfo ("id", 0, typeof(Int32)), new ColumnInfo ("time", 1, typeof(DateTime))),
|
||||
myCollection = new CircularRowCollection(
|
||||
new BlueTableSchema("Time",
|
||||
new ColumnInfo("id", 0, typeof(Int32)), new ColumnInfo("time", 1, typeof(DateTime))),
|
||||
mySizeOfStream);
|
||||
}
|
||||
}
|
||||
|
||||
#region ViewStream overriden methods
|
||||
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
myDb.addWorker (this);
|
||||
myInit ();
|
||||
protected override void myOnFirstSubscriber() {
|
||||
myDb.addWorker(this);
|
||||
myInit();
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
myDb.removeWorker (this);
|
||||
protected override void myOnLastSubscriber() {
|
||||
myDb.removeWorker(this);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
protected virtual void myInit ()
|
||||
{
|
||||
protected virtual void myInit() {
|
||||
int periods = (int)
|
||||
(DateTime.Now - myConstructionTime).TotalMilliseconds / (int) myOffset.TotalMilliseconds;
|
||||
|
||||
(DateTime.Now - myConstructionTime).TotalMilliseconds / (int)myOffset.TotalMilliseconds;
|
||||
|
||||
DateTime dt = myGetLatestRow() == null ? myRoundTime(myZeroTime) :
|
||||
myRoundTime (myZeroTime + new TimeSpan(myOffset.Ticks * periods));
|
||||
|
||||
DateTime dt = myGetLatestRow() == null ? myRoundTime(myZeroTime) :
|
||||
myRoundTime(myZeroTime + new TimeSpan(myOffset.Ticks * periods));
|
||||
|
||||
|
||||
if (myGetLatestRow() == null ||
|
||||
!myGetLatestRow()["time"].Equals(dt))
|
||||
{
|
||||
IBlueRow ar = new BlueRow (myCollection.schema,
|
||||
!myGetLatestRow()["time"].Equals(dt)) {
|
||||
IBlueRow ar = new BlueRow(myCollection.schema,
|
||||
myId++, dt);
|
||||
IBlueRow dr = myCollection.addRowWithReturn (ar);
|
||||
myFireEvent (new DataPacket (myCollection,
|
||||
myConvertToCollection (ar),
|
||||
myConvertToCollection (dr)));
|
||||
IBlueRow dr = myCollection.addRowWithReturn(ar);
|
||||
if (n_updates <= 3000)
|
||||
myFireEvent(new DataPacket(myCollection,
|
||||
myConvertToCollection(ar),
|
||||
myConvertToCollection(dr)));
|
||||
}
|
||||
}
|
||||
|
||||
#region IWorker methods
|
||||
|
||||
public void doWork ()
|
||||
{
|
||||
myInit ();
|
||||
public void doWork() {
|
||||
myInit();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
protected abstract DateTime myRoundTime (DateTime dt);
|
||||
|
||||
protected abstract DateTime myRoundTime(DateTime dt);
|
||||
|
||||
|
||||
#region Private Helper methods
|
||||
|
||||
private IBlueRow myGetLatestRow ()
|
||||
{
|
||||
private IBlueRow myGetLatestRow() {
|
||||
return myCollection.Count == 0 ?
|
||||
null :
|
||||
BlueDataUtility.extractRow (myCollection, myCollection.Count - 1);
|
||||
BlueDataUtility.extractRow(myCollection, myCollection.Count - 1);
|
||||
}
|
||||
|
||||
private IBlueRowCollection myConvertToCollection (IBlueRow r)
|
||||
{
|
||||
BlueTable retVal = new BlueTable (myCollection.schema);
|
||||
private IBlueRowCollection myConvertToCollection(IBlueRow r) {
|
||||
BlueTable retVal = new BlueTable(myCollection.schema);
|
||||
if (r != null)
|
||||
retVal.addRow (r);
|
||||
retVal.addRow(r);
|
||||
return retVal;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq.Expressions;
|
||||
using System.Xml.Serialization;
|
||||
|
||||
using BCG.Utilities;
|
||||
|
||||
@@ -53,7 +55,12 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public virtual void createTable(IBlueRowCollection newTable) {
|
||||
bts.Log.Gen.info($"PubTableBroker.createTable table: {newTable.schema.tableName}");
|
||||
string log_msg = $"PubTableBroker.createTable table: {newTable.schema.tableName}\n";
|
||||
foreach (IBlueRow r in newTable) {
|
||||
log_msg += $"id: {r["id"]}";
|
||||
}
|
||||
bts.Log.Gen.info(log_msg);
|
||||
_check_schema_consistency(newTable.schema);
|
||||
myCheckTransaction();
|
||||
IPubTablePublisher table_publisher = null;
|
||||
var name_key = newTable.schema.tableName.ToLower();
|
||||
@@ -71,6 +78,9 @@ namespace BCG {
|
||||
/// </summary>
|
||||
/// <param name="r"></param>
|
||||
public virtual void addRow(IBlueRow r) {
|
||||
string log_msg = $"PubTableBroker.addRow table: {r.schema.tableName} id: {r["id"]}";
|
||||
bts.Log.Gen.info(log_msg);
|
||||
_check_schema_consistency(r.schema);
|
||||
myCheckTransaction();
|
||||
IPubTablePublisher table_publisher = null;
|
||||
var name_key = r.schema.tableName.ToLower();
|
||||
@@ -86,6 +96,9 @@ namespace BCG {
|
||||
/// </summary>
|
||||
/// <param name="r"></param>
|
||||
public virtual void deleteRow(IBlueRow r) {
|
||||
string log_msg = $"PubTableBroker.deleteRow table: {r.schema.tableName} id: {r["id"]}";
|
||||
bts.Log.Gen.info(log_msg);
|
||||
_check_schema_consistency(r.schema);
|
||||
myCheckTransaction();
|
||||
IPubTablePublisher table_publisher = null;
|
||||
var name_key = r.schema.tableName.ToLower();
|
||||
@@ -101,6 +114,9 @@ namespace BCG {
|
||||
/// </summary>
|
||||
/// <param name="r"></param>
|
||||
public virtual void changeRow(IBlueRow r) {
|
||||
string log_msg = $"PubTableBroker.changeRow table: {r.schema.tableName} id: {r["id"]}";
|
||||
bts.Log.Gen.info(log_msg);
|
||||
_check_schema_consistency(r.schema);
|
||||
myCheckTransaction();
|
||||
IPubTablePublisher table_publisher = null;
|
||||
var name_key = r.schema.tableName.ToLower();
|
||||
@@ -118,7 +134,16 @@ namespace BCG {
|
||||
public virtual void makeBatchChanges(IBlueRowCollection newRows,
|
||||
IBlueRowCollection deletedRows,
|
||||
IBlueRowCollection addedRows) {
|
||||
myCheckTransaction();
|
||||
string log_msg = $"PubTableBroker.makeBatchChanges {deletedRows.Count} {addedRows.Count} table: {newRows.schema.tableName} deleted_rows: ";
|
||||
foreach (IBlueRow r in deletedRows)
|
||||
log_msg += $"id: {r["id"]} ";
|
||||
log_msg += "addedRows: ";
|
||||
foreach (IBlueRow r in addedRows)
|
||||
log_msg += $"id: {r["id"]}";
|
||||
bts.Log.Gen.info(log_msg);
|
||||
_check_schema_consistency(newRows.schema);
|
||||
_check_schema_consistency(deletedRows.schema);
|
||||
_check_schema_consistency(addedRows.schema);
|
||||
IPubTablePublisher table_publisher = null;
|
||||
var name_key = newRows.schema.tableName.ToLower();
|
||||
if (!_publisher_map.TryGetValue(name_key, out table_publisher))
|
||||
@@ -144,8 +169,16 @@ namespace BCG {
|
||||
}
|
||||
|
||||
protected virtual IPubTablePublisher _create_pubtable_publisher(string tag) {
|
||||
bts.Log.Gen.info($"PubTableBroker._create_pubtable_publisher tag: {tag}");
|
||||
return new PubTablePublisher(tag, _publisher_core);
|
||||
}
|
||||
|
||||
private void _check_schema_consistency(IBlueTableSchema schema_arg) {
|
||||
IBlueTableSchema schema;
|
||||
if (_schema_map.TryGetValue(schema_arg.tableName, out schema)) {
|
||||
bts.Log.assert(BlueDataUtility.compareSchemas(schema, schema_arg));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,7 @@ namespace BCG {
|
||||
/// is not uniquely chosen)
|
||||
public ViewPublisher(string name, PubTableBroker br, ViewStream vs) {
|
||||
myName = name;
|
||||
myView = vs;//String.IsNullOrEmpty(name) ? vs : new ProjectStream(vs, "!", myName);
|
||||
myView = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
|
||||
myBroker = br;
|
||||
myFirstTime = true;
|
||||
myView.onChange += new DataStream.ChangeDelegate(myChangeHandler);
|
||||
|
||||
Reference in new issue
Block a user