TimeSeries and TimeBucketStreams, Stream offloading etc
This commit is contained in:
1 parent
33030b1212
commit
88bf04e5f1
31 files changed
+1823
-836
No files matched your search
@@ -0,0 +1,24 @@
|
||||
using BCG;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
|
||||
[DataStreamUserRespresentation("add_to_cloud")]
|
||||
public class AddToCloudStream : ViewStream
|
||||
{
|
||||
private readonly string _name, _stream_exp;
|
||||
private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
|
||||
public AddToCloudStream(string name, string stream_exp)
|
||||
: this(name, stream_exp, 0)
|
||||
{
|
||||
}
|
||||
|
||||
public AddToCloudStream(string name, string stream_exp, int persist)
|
||||
{
|
||||
_name = name;
|
||||
_stream_exp = stream_exp;
|
||||
_requestor.add_to_cloud(_name, _stream_exp, persist == 1);
|
||||
mySetRealSource(new TableStream("cloud_streams"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
using mbts.config;
|
||||
using bts.utils;
|
||||
using System.Threading.Tasks;
|
||||
using Bts.DataStreamExpressions;
|
||||
using System.Collections.Generic;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
// listens to the requests posted on "cloud_streams" request table, and
|
||||
// if unfulfilled starts a ViewPubsliher
|
||||
public class CloudStreamManager : IWorker
|
||||
{
|
||||
public class StreamPubUnit
|
||||
{
|
||||
public string name;
|
||||
public string stream_expression;
|
||||
public ViewStream stream;
|
||||
public bool persist;
|
||||
public ViewPublisher view_publisher;
|
||||
|
||||
public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
|
||||
{
|
||||
name = name_;
|
||||
stream_expression = stream_expression_;
|
||||
stream = stream_;
|
||||
persist = persist_;
|
||||
view_publisher = view_publisher_;
|
||||
}
|
||||
}
|
||||
static ViewStream build_stream(string stream_expression)
|
||||
{
|
||||
try
|
||||
{
|
||||
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
|
||||
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||
return rv;
|
||||
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
private IDictionary<string, StreamPubUnit>
|
||||
_stream_pub_map = new Dictionary<string, StreamPubUnit>();
|
||||
private ViewStream _stream = new TableStream("cloud_streams");
|
||||
private PubTableBroker _broker = new PubTableBroker();
|
||||
|
||||
public CloudStreamManager()
|
||||
{
|
||||
_stream.onChange += _on_str;
|
||||
|
||||
}
|
||||
private void _on_str(object obj)
|
||||
{
|
||||
DataPacket dp = obj as DataPacket;
|
||||
foreach (IBlueRow r in dp.newRows)
|
||||
{
|
||||
string name = r.get<string>("name");
|
||||
bool persist = r.get<bool>("persist");
|
||||
string stream_exp = r.get<string>("stream_exp");
|
||||
if (!_stream_pub_map.ContainsKey(name))
|
||||
{
|
||||
_assign(name, stream_exp, persist);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void _assign(string name, string stream_expression, bool persist)
|
||||
{
|
||||
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
|
||||
ViewStream stream = build_stream(stream_expression);
|
||||
_expose_to_ws(name, stream);
|
||||
if (persist)
|
||||
{
|
||||
stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs
|
||||
}
|
||||
ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
|
||||
StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
|
||||
_stream_pub_map.Add(name, u);
|
||||
}
|
||||
|
||||
private void _expose_to_ws(string name, ViewStream stream)
|
||||
{
|
||||
// TODO: john
|
||||
}
|
||||
|
||||
public void doWork()
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
using BCG;
|
||||
|
||||
using System.Linq;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class StreamOffloadRequestor
|
||||
{
|
||||
public class OffloadOp
|
||||
{
|
||||
string name;
|
||||
string stream_exp;
|
||||
}
|
||||
private IBlueTableSchema _schema;
|
||||
private IBlueRowCollection _rows;
|
||||
private PubTableBroker _broker = new PubTableBroker();
|
||||
ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
|
||||
public StreamOffloadRequestor()
|
||||
{
|
||||
_schema = new BlueTableSchema("cloud_streams",
|
||||
new ColumnInfo("id", 0, typeof(int)),
|
||||
new ColumnInfo("name", 1, typeof(string)),
|
||||
new ColumnInfo("persist", 2, typeof(bool)),
|
||||
new ColumnInfo("stream_exp", 3, typeof(string)));
|
||||
_existing_offloads_vs.onChange += _on_str;
|
||||
}
|
||||
private void _on_str(object obj)
|
||||
{
|
||||
DataPacket dp = obj as DataPacket;
|
||||
_rows = dp.newRows;
|
||||
// this is needed so the broker knows the state of the table
|
||||
_broker.createTable(_rows);
|
||||
}
|
||||
public void add_to_cloud(string name, string stream_exp, bool persist)
|
||||
{
|
||||
// check if it exists
|
||||
if (_rows == null)
|
||||
{
|
||||
BlueTable table = new BlueTable(_schema);
|
||||
IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
|
||||
table.addRow(nr);
|
||||
_broker.createTable(table);
|
||||
// create table
|
||||
}
|
||||
else
|
||||
{
|
||||
if (exists(name))
|
||||
return;
|
||||
IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
|
||||
_broker.addRow(nr);
|
||||
}
|
||||
}
|
||||
|
||||
bool exists(string name)
|
||||
{
|
||||
return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ using mbts.spark;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
[DataStreamUserRespresentation("fastJoin")]
|
||||
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
|
||||
{
|
||||
private readonly NonIncJoinSpec[] _join_specs;
|
||||
@@ -23,7 +24,7 @@ namespace BCG
|
||||
|
||||
public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
|
||||
ViewStream right, string right_join_cols, string right_proj)
|
||||
: this(new Triplet<ViewStream, string, string>[]
|
||||
: this(new Triplet<ViewStream, string, string>[]
|
||||
{
|
||||
new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
|
||||
new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
|
||||
@@ -31,7 +32,7 @@ namespace BCG
|
||||
{ }
|
||||
|
||||
public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
|
||||
: this (NonIncJoinSpec.from_triplet (join_specs), null, 0)
|
||||
: this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
|
||||
{ }
|
||||
|
||||
public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
|
||||
@@ -102,8 +103,8 @@ namespace BCG
|
||||
{
|
||||
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
|
||||
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
||||
d.constribute_cols_to_schema (cis);
|
||||
_output_schema = new BlueTableSchema ("NonIncJoinStream", cis.ToArray());
|
||||
d.constribute_cols_to_schema(cis);
|
||||
_output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,8 @@ namespace BCG {
|
||||
|
||||
private void _onstream_change(object o) {
|
||||
DataPacket dp = o as DataPacket;
|
||||
BlueDataUtility.validate_table_schema(dp.newRows);
|
||||
BlueDataUtility.validate_table_schema(dp.addedRows);
|
||||
if (_last_recv_rows == null) {
|
||||
_last_recv_rows = dp.newRows;
|
||||
} else {
|
||||
|
||||
@@ -1,37 +1,47 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
|
||||
using System;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
namespace BCG {
|
||||
namespace BCG
|
||||
{
|
||||
// Keeps adding changing rows to the stream, hence preserving the history
|
||||
// and creating a time series, for example this can be used to make bbo updates
|
||||
// into a time series of prices/qty
|
||||
public class TimeSeriesStream : ViewStream {
|
||||
public class TimeSeriesStream : ViewStream
|
||||
{
|
||||
private BlueTable _table;
|
||||
private ViewStream _stream;
|
||||
private int _id = 0;
|
||||
|
||||
public TimeSeriesStream(ViewStream stream) {
|
||||
public TimeSeriesStream(ViewStream stream)
|
||||
{
|
||||
_stream = stream;
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
_stream.onChange += new ChangeDelegate(_onstream_change);
|
||||
}
|
||||
protected override void myOnLastSubscriber() {
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
_stream.onChange -= new ChangeDelegate(_onstream_change);
|
||||
}
|
||||
|
||||
private void _onstream_change(object o) {
|
||||
|
||||
private void _onstream_change(object o)
|
||||
{
|
||||
DataPacket dp = o as DataPacket;
|
||||
DateTime now = DateTime.Now;
|
||||
if (_table == null) {
|
||||
if (_table == null)
|
||||
{
|
||||
IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
|
||||
schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
|
||||
_table = new BlueTable(schema);
|
||||
}
|
||||
BlueTable added_rows = new BlueTable(_table.schema);
|
||||
foreach (IBlueRow r in dp.addedRows) {
|
||||
foreach (IBlueRow r in dp.addedRows)
|
||||
{
|
||||
MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
|
||||
ar.setValue("timestamp", now);
|
||||
ar.setValue("id", _id++);
|
||||
@@ -42,4 +52,118 @@ namespace BCG {
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class TimeBucketStream : ViewStream, IStreamMapper
|
||||
{
|
||||
private ViewStream _src;
|
||||
private readonly string _time_stamp_col;
|
||||
private IBlueTableSchema _schema;
|
||||
private int _bucket_size_seconds;
|
||||
|
||||
public TimeBucketStream(ViewStream input, string time_stamp_col, int bucket_size_seconds)
|
||||
{
|
||||
_time_stamp_col = time_stamp_col;
|
||||
_bucket_size_seconds = bucket_size_seconds;
|
||||
var vs = new MapStream(input, this);
|
||||
mySetRealSource(vs);
|
||||
}
|
||||
|
||||
#region IStreamMapper Members
|
||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema)
|
||||
{
|
||||
IBlueTableSchema rv = input_schema;
|
||||
//rv = BlueDataUtility.remove_columns(rv, _time_stamp_col);
|
||||
rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int));
|
||||
return rv;
|
||||
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region IDynamicStreamMapper Members
|
||||
|
||||
public IBlueRow map(IBlueRow row)
|
||||
{
|
||||
DateTime ts = row.get<DateTime>(_time_stamp_col);
|
||||
TimeSpan time_of_day = (ts - ts.Date);
|
||||
int seconds = Convert.ToInt32(time_of_day.TotalSeconds);
|
||||
double buckets = seconds / _bucket_size_seconds;
|
||||
int bucket_number = Convert.ToInt32(Math.Floor(buckets));
|
||||
MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
|
||||
rv.setValue("bucket_number", bucket_number);
|
||||
return rv;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
// public class ThrottleStream : ViewStream, IWorker
|
||||
// {
|
||||
// private ViewStreamDatabase _vsd;
|
||||
// private ViewStream _stream;
|
||||
// private int _throttle_ms;
|
||||
// private TimeSpan _min_fire_period;
|
||||
// private DateTime _last_Fire_time = DateTime.MinValue;
|
||||
// private DataPacket _unfired_dp;
|
||||
// public ThrottleStream(ViewStreamDatabase vsd, ViewStream base_stream, int throttle_ms)
|
||||
// {
|
||||
// _vsd = vsd;
|
||||
// _stream = base_stream;
|
||||
// _throttle_ms = throttle_ms;
|
||||
// _min_fire_period = TimeSpan.FromMilliSeconds(throttle_ms);
|
||||
// }
|
||||
|
||||
// protected override void myOnFirstSubscriber()
|
||||
// {
|
||||
// _vsd.addWorker(this);
|
||||
// _stream.onChange += new ChangeDelegate(_onstream_change);
|
||||
// }
|
||||
// protected override void myOnLastSubscriber()
|
||||
// {
|
||||
// _vsd.removeWorker(this);
|
||||
// _stream.onChange -= new ChangeDelegate(_onstream_change);
|
||||
// }
|
||||
|
||||
// private void _onstream_change(object obj)
|
||||
// {
|
||||
// DataPacket dp = obj as DataPacket;
|
||||
// if (DateTime.Now - _last_Fire_time >= _min_fire_period)
|
||||
// {
|
||||
// // ok to fire here
|
||||
// }
|
||||
// else
|
||||
// {
|
||||
// _last_cached_state = dp.newRows;
|
||||
// }
|
||||
// _possibly_fire(dp.newRows);
|
||||
// }
|
||||
|
||||
// public void doWork()
|
||||
// {
|
||||
// if (_unfired_dp != null)
|
||||
// {
|
||||
|
||||
// }
|
||||
// }
|
||||
|
||||
// private void _possibly_fire()
|
||||
// {
|
||||
// if (_unfired_dp != null)
|
||||
// {
|
||||
// if (DateTime.Now - _last_fire_time > _min_fire_period)
|
||||
// {
|
||||
// DataPacket dp = BlueDataUtility.build_dp_inc(_unfired_dp.newRows, myLastValue);
|
||||
// _unfired_dp = null;
|
||||
// _last_Fire_time = DateTime.Now;
|
||||
// myFireEvent(dp);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// }
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ using BCG.Utilities;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
[DataStreamUserRespresentation("csvSaver")]
|
||||
[DataStreamUserRespresentation("CSVSaver")]
|
||||
public class CSVSaverStream : ViewStream, ISideEffectStream
|
||||
{
|
||||
private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
|
||||
|
||||
Whitespace-only changes.
@@ -0,0 +1,51 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using System.Collections;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
[DataStreamUserRepresentation("timeBucket")]
|
||||
public class TimeBucketStream : ViewStream, IStreamMapper
|
||||
{
|
||||
private ViewStream _src;
|
||||
private readonly string _time_stamp_col;
|
||||
private IBlueTableSchema _schema;
|
||||
private uint _bucket_size_seconds;
|
||||
|
||||
public TimeBucketStream(ViewStream input, string time_stamp_col, uint bucket_size_seconds)
|
||||
{
|
||||
_time_stamp_col = time_stamp_col;
|
||||
_bucket_size_seconds = bucket_size_seconds;
|
||||
}
|
||||
|
||||
#region IStreamMapper Members
|
||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema)
|
||||
{
|
||||
IBlueTableSchema rv = input_schema;
|
||||
//rv = BlueDataUtility.remove_columns(rv, _time_stamp_col);
|
||||
rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int));
|
||||
return rv;
|
||||
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region IDynamicStreamMapper Members
|
||||
|
||||
public IBlueRow map(IBlueRow row)
|
||||
{
|
||||
DateTime ts = row.get<DateTime>(_time_stamp_col);
|
||||
TimeSpan time_of_day = (ts - ts.Date)
|
||||
int seconds = Convert.ToInt32(time_of_day.TotalSeconds);
|
||||
int buckets = seconds / _bucket_size_seconds;
|
||||
int bucket_number = Math.Floor(buckets);
|
||||
MutableBlueRow rv = new MutableBlueRow(row, getOutputSchema(row.schema))
|
||||
rv.setValue("bucket_number", bucket_number);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,9 @@
|
||||
using System;
|
||||
|
||||
using BCG.Utilities.General;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public abstract class TimeUnitStream : ViewStream, IWorker
|
||||
{
|
||||
namespace BCG {
|
||||
public abstract class TimeUnitStream : ViewStream, IWorker {
|
||||
protected TimeSpan myOffset;
|
||||
protected DateTime myConstructionTime;
|
||||
protected int mySizeOfStream;
|
||||
@@ -13,87 +12,80 @@ namespace BCG
|
||||
protected ViewStreamDatabase myDb;
|
||||
protected int myId = 0;
|
||||
|
||||
public TimeUnitStream (DateTime zeroTime, TimeSpan offSet, int sizeOfStream, ViewStreamDatabase db)
|
||||
{
|
||||
public TimeUnitStream(DateTime zeroTime, TimeSpan offSet, int sizeOfStream, ViewStreamDatabase db) {
|
||||
myConstructionTime = DateTime.Now;
|
||||
myZeroTime = zeroTime;
|
||||
myOffset = offSet;
|
||||
mySizeOfStream = sizeOfStream;
|
||||
myDb = db;
|
||||
myCollection = new CircularRowCollection (
|
||||
new BlueTableSchema ("Time",
|
||||
new ColumnInfo ("id", 0, typeof(Int32)), new ColumnInfo ("time", 1, typeof(DateTime))),
|
||||
myCollection = new CircularRowCollection(
|
||||
new BlueTableSchema("Time",
|
||||
new ColumnInfo("id", 0, typeof(Int32)), new ColumnInfo("time", 1, typeof(DateTime))),
|
||||
mySizeOfStream);
|
||||
}
|
||||
}
|
||||
|
||||
#region ViewStream overriden methods
|
||||
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
myDb.addWorker (this);
|
||||
myInit ();
|
||||
protected override void myOnFirstSubscriber() {
|
||||
myDb.addWorker(this);
|
||||
myInit();
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
myDb.removeWorker (this);
|
||||
protected override void myOnLastSubscriber() {
|
||||
myDb.removeWorker(this);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
protected virtual void myInit ()
|
||||
{
|
||||
protected virtual void myInit() {
|
||||
int periods = (int)
|
||||
(DateTime.Now - myConstructionTime).TotalMilliseconds / (int) myOffset.TotalMilliseconds;
|
||||
|
||||
(DateTime.Now - myConstructionTime).TotalMilliseconds / (int)myOffset.TotalMilliseconds;
|
||||
|
||||
DateTime dt = myGetLatestRow() == null ? myRoundTime(myZeroTime) :
|
||||
myRoundTime (myZeroTime + new TimeSpan(myOffset.Ticks * periods));
|
||||
|
||||
DateTime dt = myGetLatestRow() == null ? myRoundTime(myZeroTime) :
|
||||
myRoundTime(myZeroTime + new TimeSpan(myOffset.Ticks * periods));
|
||||
|
||||
|
||||
if (myGetLatestRow() == null ||
|
||||
!myGetLatestRow()["time"].Equals(dt))
|
||||
{
|
||||
IBlueRow ar = new BlueRow (myCollection.schema,
|
||||
!myGetLatestRow()["time"].Equals(dt)) {
|
||||
IBlueRow ar = new BlueRow(myCollection.schema,
|
||||
myId++, dt);
|
||||
IBlueRow dr = myCollection.addRowWithReturn (ar);
|
||||
myFireEvent (new DataPacket (myCollection,
|
||||
myConvertToCollection (ar),
|
||||
myConvertToCollection (dr)));
|
||||
IBlueRow dr = myCollection.addRowWithReturn(ar);
|
||||
if (n_updates <= 3000)
|
||||
myFireEvent(new DataPacket(myCollection,
|
||||
myConvertToCollection(ar),
|
||||
myConvertToCollection(dr)));
|
||||
}
|
||||
}
|
||||
|
||||
#region IWorker methods
|
||||
|
||||
public void doWork ()
|
||||
{
|
||||
myInit ();
|
||||
public void doWork() {
|
||||
myInit();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
protected abstract DateTime myRoundTime (DateTime dt);
|
||||
|
||||
protected abstract DateTime myRoundTime(DateTime dt);
|
||||
|
||||
|
||||
#region Private Helper methods
|
||||
|
||||
private IBlueRow myGetLatestRow ()
|
||||
{
|
||||
private IBlueRow myGetLatestRow() {
|
||||
return myCollection.Count == 0 ?
|
||||
null :
|
||||
BlueDataUtility.extractRow (myCollection, myCollection.Count - 1);
|
||||
BlueDataUtility.extractRow(myCollection, myCollection.Count - 1);
|
||||
}
|
||||
|
||||
private IBlueRowCollection myConvertToCollection (IBlueRow r)
|
||||
{
|
||||
BlueTable retVal = new BlueTable (myCollection.schema);
|
||||
private IBlueRowCollection myConvertToCollection(IBlueRow r) {
|
||||
BlueTable retVal = new BlueTable(myCollection.schema);
|
||||
if (r != null)
|
||||
retVal.addRow (r);
|
||||
retVal.addRow(r);
|
||||
return retVal;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq.Expressions;
|
||||
using System.Xml.Serialization;
|
||||
|
||||
using BCG.Utilities;
|
||||
|
||||
@@ -53,7 +55,12 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public virtual void createTable(IBlueRowCollection newTable) {
|
||||
bts.Log.Gen.info($"PubTableBroker.createTable table: {newTable.schema.tableName}");
|
||||
string log_msg = $"PubTableBroker.createTable table: {newTable.schema.tableName}\n";
|
||||
foreach (IBlueRow r in newTable) {
|
||||
log_msg += $"id: {r["id"]}";
|
||||
}
|
||||
bts.Log.Gen.info(log_msg);
|
||||
_check_schema_consistency(newTable.schema);
|
||||
myCheckTransaction();
|
||||
IPubTablePublisher table_publisher = null;
|
||||
var name_key = newTable.schema.tableName.ToLower();
|
||||
@@ -71,6 +78,9 @@ namespace BCG {
|
||||
/// </summary>
|
||||
/// <param name="r"></param>
|
||||
public virtual void addRow(IBlueRow r) {
|
||||
string log_msg = $"PubTableBroker.addRow table: {r.schema.tableName} id: {r["id"]}";
|
||||
bts.Log.Gen.info(log_msg);
|
||||
_check_schema_consistency(r.schema);
|
||||
myCheckTransaction();
|
||||
IPubTablePublisher table_publisher = null;
|
||||
var name_key = r.schema.tableName.ToLower();
|
||||
@@ -86,6 +96,9 @@ namespace BCG {
|
||||
/// </summary>
|
||||
/// <param name="r"></param>
|
||||
public virtual void deleteRow(IBlueRow r) {
|
||||
string log_msg = $"PubTableBroker.deleteRow table: {r.schema.tableName} id: {r["id"]}";
|
||||
bts.Log.Gen.info(log_msg);
|
||||
_check_schema_consistency(r.schema);
|
||||
myCheckTransaction();
|
||||
IPubTablePublisher table_publisher = null;
|
||||
var name_key = r.schema.tableName.ToLower();
|
||||
@@ -101,6 +114,9 @@ namespace BCG {
|
||||
/// </summary>
|
||||
/// <param name="r"></param>
|
||||
public virtual void changeRow(IBlueRow r) {
|
||||
string log_msg = $"PubTableBroker.changeRow table: {r.schema.tableName} id: {r["id"]}";
|
||||
bts.Log.Gen.info(log_msg);
|
||||
_check_schema_consistency(r.schema);
|
||||
myCheckTransaction();
|
||||
IPubTablePublisher table_publisher = null;
|
||||
var name_key = r.schema.tableName.ToLower();
|
||||
@@ -118,7 +134,16 @@ namespace BCG {
|
||||
public virtual void makeBatchChanges(IBlueRowCollection newRows,
|
||||
IBlueRowCollection deletedRows,
|
||||
IBlueRowCollection addedRows) {
|
||||
myCheckTransaction();
|
||||
string log_msg = $"PubTableBroker.makeBatchChanges {deletedRows.Count} {addedRows.Count} table: {newRows.schema.tableName} deleted_rows: ";
|
||||
foreach (IBlueRow r in deletedRows)
|
||||
log_msg += $"id: {r["id"]} ";
|
||||
log_msg += "addedRows: ";
|
||||
foreach (IBlueRow r in addedRows)
|
||||
log_msg += $"id: {r["id"]}";
|
||||
bts.Log.Gen.info(log_msg);
|
||||
_check_schema_consistency(newRows.schema);
|
||||
_check_schema_consistency(deletedRows.schema);
|
||||
_check_schema_consistency(addedRows.schema);
|
||||
IPubTablePublisher table_publisher = null;
|
||||
var name_key = newRows.schema.tableName.ToLower();
|
||||
if (!_publisher_map.TryGetValue(name_key, out table_publisher))
|
||||
@@ -144,8 +169,16 @@ namespace BCG {
|
||||
}
|
||||
|
||||
protected virtual IPubTablePublisher _create_pubtable_publisher(string tag) {
|
||||
bts.Log.Gen.info($"PubTableBroker._create_pubtable_publisher tag: {tag}");
|
||||
return new PubTablePublisher(tag, _publisher_core);
|
||||
}
|
||||
|
||||
private void _check_schema_consistency(IBlueTableSchema schema_arg) {
|
||||
IBlueTableSchema schema;
|
||||
if (_schema_map.TryGetValue(schema_arg.tableName, out schema)) {
|
||||
bts.Log.assert(BlueDataUtility.compareSchemas(schema, schema_arg));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,7 @@ namespace BCG {
|
||||
/// is not uniquely chosen)
|
||||
public ViewPublisher(string name, PubTableBroker br, ViewStream vs) {
|
||||
myName = name;
|
||||
myView = vs;//String.IsNullOrEmpty(name) ? vs : new ProjectStream(vs, "!", myName);
|
||||
myView = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
|
||||
myBroker = br;
|
||||
myFirstTime = true;
|
||||
myView.onChange += new DataStream.ChangeDelegate(myChangeHandler);
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -5,14 +5,12 @@ using System.Data;
|
||||
//using BCG.Utilities;
|
||||
using System.Linq;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// Anologue of the dotNet DataTable.
|
||||
/// </summary>
|
||||
[Serializable]
|
||||
public class BlueTable : IBlueRowCollection
|
||||
{
|
||||
public class BlueTable : IBlueRowCollection {
|
||||
/// <summary>
|
||||
/// Hashtable indexed on "id" to speed up removes
|
||||
/// </summary>
|
||||
@@ -29,22 +27,19 @@ namespace BCG
|
||||
/// all the data over
|
||||
/// </summary>
|
||||
/// <param name="table"></param>
|
||||
public BlueTable(DataTable table)
|
||||
{
|
||||
public BlueTable(DataTable table) {
|
||||
ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
|
||||
int i = 0;
|
||||
foreach (DataColumn col in table.Columns)
|
||||
cis[i] = new ColumnInfo(col.ColumnName, i++, col.DataType);
|
||||
mySchema = new BlueTableSchema(cis, table.TableName);
|
||||
foreach (DataRow row in table.Rows)
|
||||
{
|
||||
foreach (DataRow row in table.Rows) {
|
||||
IBlueRow newRow = new BlueRow(mySchema, row.ItemArray);
|
||||
myIdIndex[newRow["id"]] = newRow;
|
||||
}
|
||||
}
|
||||
|
||||
public BlueTable(IBlueRowCollection table)
|
||||
{
|
||||
public BlueTable(IBlueRowCollection table) {
|
||||
mySchema = table.schema;
|
||||
foreach (IBlueRow row in table)
|
||||
addRow(row);
|
||||
@@ -52,12 +47,10 @@ namespace BCG
|
||||
|
||||
public BlueTable(string tableName,
|
||||
IEnumerable objects,
|
||||
params CMap[] columnMappers)
|
||||
{
|
||||
params CMap[] columnMappers) {
|
||||
// extract schema from mappers
|
||||
ColumnInfo[] cols = new ColumnInfo[columnMappers.Length];
|
||||
for (int i = 0; i < columnMappers.Length; i++)
|
||||
{
|
||||
for (int i = 0; i < columnMappers.Length; i++) {
|
||||
cols[i] = new ColumnInfo(
|
||||
columnMappers[i].getColumnName(), i,
|
||||
columnMappers[i].getColumnType());
|
||||
@@ -66,8 +59,7 @@ namespace BCG
|
||||
|
||||
// load rows
|
||||
int rowNumber = 0;
|
||||
foreach (object rowObject in objects)
|
||||
{
|
||||
foreach (object rowObject in objects) {
|
||||
object[] values = new object[columnMappers.Length];
|
||||
for (int cn = 0; cn < columnMappers.Length; cn++)
|
||||
values[cn] = columnMappers[cn].getColumnValue(rowNumber, rowObject);
|
||||
@@ -87,10 +79,8 @@ namespace BCG
|
||||
/// <summary>
|
||||
/// Returns the row by the index, interface property
|
||||
/// </summary>
|
||||
public virtual IBlueRow this[int id]
|
||||
{
|
||||
get
|
||||
{
|
||||
public virtual IBlueRow this[int id] {
|
||||
get {
|
||||
object row = myIdIndex[id];
|
||||
if (row == null)
|
||||
throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
|
||||
@@ -105,8 +95,7 @@ namespace BCG
|
||||
|
||||
public virtual bool containsID(int id) { return myIdIndex[id] != null; }
|
||||
|
||||
public virtual bool tryGetValue(int id, out IBlueRow row)
|
||||
{
|
||||
public virtual bool tryGetValue(int id, out IBlueRow row) {
|
||||
row = (IBlueRow)myIdIndex[id];
|
||||
return (row != null);
|
||||
}
|
||||
@@ -114,8 +103,7 @@ namespace BCG
|
||||
#endregion
|
||||
|
||||
#region BlueTable members
|
||||
public virtual void clear()
|
||||
{
|
||||
public virtual void clear() {
|
||||
myIdIndex.Clear();
|
||||
}
|
||||
|
||||
@@ -123,8 +111,7 @@ namespace BCG
|
||||
/// If a row with the same id exists, it replaces it
|
||||
/// </summary>
|
||||
/// <param name="r"></param>
|
||||
public virtual void forceAddRow(IBlueRow r)
|
||||
{
|
||||
public virtual void forceAddRow(IBlueRow r) {
|
||||
myIdIndex[r["id"]] = r;
|
||||
}
|
||||
|
||||
@@ -132,43 +119,41 @@ namespace BCG
|
||||
/// Adds a new row to the table.
|
||||
/// </summary>
|
||||
/// <param name="row"></param>
|
||||
public virtual void addRow(IBlueRow row)
|
||||
{
|
||||
public virtual void addRow(IBlueRow row) {
|
||||
object id = row["id"];
|
||||
|
||||
if (id == null || id == DBNull.Value)
|
||||
throw new BlueDataException("The ID of the row being added is null/DNBull");
|
||||
try
|
||||
{
|
||||
try {
|
||||
myIdIndex.Add(id, row);
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
throw new BlueDataException($"ID {id} already exists in the table: {mySchema.tableName} , duplication of field ID not allowed");
|
||||
} catch (ArgumentException) {
|
||||
throw new BlueDataException($"ID: {id} already exists in the table: {mySchema.tableName}");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public virtual void addRow(DataRow row)
|
||||
{
|
||||
public virtual void addRow(DataRow row) {
|
||||
object val = row["id"];
|
||||
|
||||
if (val == null || val == DBNull.Value)
|
||||
throw new BlueDataException("The ID of the row being added is null/DNBull");
|
||||
if (myIdIndex[val] != null)
|
||||
throw new BlueDataException("ID already exists in the table, duplication of field ID not allowed");
|
||||
if (myIdIndex[val] != null) {
|
||||
string err_msg = $"ID: {val} already exists in the table: {mySchema.tableName} duplication of field ID not allowed";
|
||||
// temp TODOds remove it
|
||||
bts.Log.assert(false, err_msg);
|
||||
throw new BlueDataException(err_msg);
|
||||
|
||||
}
|
||||
IBlueRow newRow = new BlueRow(mySchema, row);
|
||||
myIdIndex[val] = newRow;
|
||||
}
|
||||
|
||||
public virtual void addRows(IBlueRowCollection rows)
|
||||
{
|
||||
public virtual void addRows(IBlueRowCollection rows) {
|
||||
foreach (IBlueRow nr in rows)
|
||||
addRow(nr);
|
||||
}
|
||||
|
||||
public virtual void addRows(IEnumerable rows)
|
||||
{
|
||||
public virtual void addRows(IEnumerable rows) {
|
||||
foreach (IBlueRow nr in rows)
|
||||
addRow(nr);
|
||||
}
|
||||
@@ -185,37 +170,32 @@ namespace BCG
|
||||
/// Removes a row specified by the id. The index helps in making this operation O(1)
|
||||
/// </summary>
|
||||
/// <param name="id"></param>
|
||||
public virtual bool deleteRow(int id)
|
||||
{
|
||||
public virtual bool deleteRow(int id) {
|
||||
if (myIdIndex[id] == null)
|
||||
return false;
|
||||
myIdIndex.Remove(id);
|
||||
return true;
|
||||
}
|
||||
|
||||
public virtual bool deleteRows(IBlueRowCollection rows)
|
||||
{
|
||||
public virtual bool deleteRows(IBlueRowCollection rows) {
|
||||
bool retVal = true;
|
||||
foreach (IBlueRow r in rows)
|
||||
retVal &= deleteRow(r);
|
||||
return retVal;
|
||||
}
|
||||
|
||||
public virtual bool deleteRows(IEnumerable rows)
|
||||
{
|
||||
public virtual bool deleteRows(IEnumerable rows) {
|
||||
bool retVal = true;
|
||||
foreach (IBlueRow r in rows)
|
||||
retVal &= deleteRow(r);
|
||||
return retVal;
|
||||
}
|
||||
|
||||
public virtual void changeRow(IBlueRow r)
|
||||
{
|
||||
public virtual void changeRow(IBlueRow r) {
|
||||
myIdIndex[r["id"]] = r;
|
||||
}
|
||||
|
||||
public virtual void changeRows(IBlueRowCollection rows)
|
||||
{
|
||||
public virtual void changeRows(IBlueRowCollection rows) {
|
||||
foreach (IBlueRow r in rows)
|
||||
changeRow(r);
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ namespace BCG
|
||||
Delegate.Combine (myOnUpdateDelegate, value) as TableUpdateDelegate;
|
||||
if (myStateTable != null)
|
||||
value (myStateTable, myStateTable, new BlueTable (myStateTable.schema));
|
||||
}
|
||||
}
|
||||
remove
|
||||
{
|
||||
// BlueEventLog.logEvent ("TableSubscriber being Unsubscribed for tag :" + myTag, BlueEventLevel.Information);
|
||||
@@ -64,6 +64,7 @@ namespace BCG
|
||||
|
||||
public void update(Stream[] streams)
|
||||
{
|
||||
bts.Log.Gen.info($"TableAccumulator.update tag: {myTag} n_streams: {streams.Length} stream[0].len: {streams[0].Length - streams[0].Position}");
|
||||
//checking the condition as a subscriber might unsubscribe in a callback and there might be more than one
|
||||
//data element for that tag in the ViewStreamDatabase
|
||||
if (myOnUpdateDelegate != null)
|
||||
|
||||
+31
-33
@@ -3,16 +3,15 @@ using System.IO;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// Controls (ie creates dynamically) an IClientDeserializer and then passes data to it.
|
||||
/// </summary>
|
||||
public class ClientDeserializationController : IClientDeserializer
|
||||
{
|
||||
public class ClientDeserializationController : IClientDeserializer {
|
||||
private IClientDeserializer myDeserializer;
|
||||
|
||||
public ClientDeserializationController () {}
|
||||
private int _n_desers = 0;
|
||||
|
||||
public ClientDeserializationController() { }
|
||||
|
||||
#region IClientDeserializer Members
|
||||
|
||||
@@ -23,42 +22,41 @@ namespace BCG
|
||||
/// <param name="streams"></param>
|
||||
/// <param name="theOps"></param>
|
||||
/// <param name="theRows"></param>
|
||||
public void deserialize (Stream[] streams, out byte[] theOps, out Array theRows)
|
||||
{
|
||||
if (myDeserializer != null)
|
||||
{
|
||||
myDeserializer.deserialize (streams, out theOps, out theRows);
|
||||
}
|
||||
else
|
||||
{
|
||||
Stream totalStream = UtilityMethods.getAggregatedStream (streams);
|
||||
public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) {
|
||||
bts.Log.Gen.info($"ClientDeserializationController.deserialize len: {streams[0].Length - streams[0].Position}");
|
||||
if (myDeserializer != null) {
|
||||
myDeserializer.deserialize(streams, out theOps, out theRows);
|
||||
} else {
|
||||
Stream totalStream = UtilityMethods.getAggregatedStream(streams);
|
||||
long initialPos = totalStream.Position;
|
||||
BinaryReader br = new BinaryReader (totalStream);
|
||||
BinaryReader br = new BinaryReader(totalStream);
|
||||
byte op;
|
||||
try
|
||||
{
|
||||
op = br.ReadByte ();
|
||||
}
|
||||
catch (EndOfStreamException)
|
||||
{
|
||||
try {
|
||||
op = br.ReadByte();
|
||||
} catch (EndOfStreamException) {
|
||||
op = Byte.MaxValue;
|
||||
}
|
||||
if (op == StreamOpCode.create)
|
||||
{
|
||||
br.ReadString (); //pubsubclientdeserializer
|
||||
string deserName = br.ReadString ();
|
||||
myDeserializer = Activator.CreateInstance (Type.GetType (deserName))
|
||||
if (op == StreamOpCode.create) {
|
||||
br.ReadString(); //pubsubclientdeserializer
|
||||
string deserName = br.ReadString();
|
||||
myDeserializer = Activator.CreateInstance(Type.GetType(deserName))
|
||||
as IClientDeserializer;
|
||||
totalStream.Position = initialPos;
|
||||
myDeserializer.deserialize (new Stream[] {totalStream}, out theOps, out theRows);
|
||||
}
|
||||
else
|
||||
{
|
||||
myDeserializer.deserialize(new Stream[] { totalStream }, out theOps, out theRows);
|
||||
} else {
|
||||
// BlueEventLog.logEvent ("Received a non create before a create", BlueEventLevel.Warning);
|
||||
theOps = new byte[]{};
|
||||
theRows = new object[]{};
|
||||
theOps = new byte[] { };
|
||||
theRows = new object[] { };
|
||||
}
|
||||
}
|
||||
var msgs = StreamIOMessage.assemble_msgs(theOps, theRows);
|
||||
int i = 0;
|
||||
foreach (var m in msgs) {
|
||||
bts.Log.Gen.info($"ClientDeserializationController n_desers: {_n_desers} msg no: {i++}: {m}");
|
||||
}
|
||||
_n_desers++;
|
||||
// log it here
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
+32
-32
@@ -5,61 +5,61 @@ using System.IO;
|
||||
using BCG.Utilities;
|
||||
//using BCG.Utilities.BlueConfiguration;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process
|
||||
/// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations)
|
||||
/// </summary>
|
||||
internal class ClientUniversalRowDeserializer : IClientRowDeserializer
|
||||
{
|
||||
internal class ClientUniversalRowDeserializer : IClientRowDeserializer {
|
||||
private IBlueTableSchema mySchema;
|
||||
private int myNullMapLength = 0;
|
||||
private PrimitiveDeserializationHelper myDeserializationHelper;
|
||||
private BitArray myNullMap;
|
||||
private int[] mySerIndexToSchemaIndexMap;
|
||||
|
||||
public ClientUniversalRowDeserializer (IBlueTableSchema schema)
|
||||
{
|
||||
myDeserializationHelper = new PrimitiveDeserializationHelper (mySchema = schema);
|
||||
myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8)));
|
||||
myInitReorderSchema ();
|
||||
|
||||
|
||||
public ClientUniversalRowDeserializer(IBlueTableSchema schema) {
|
||||
myDeserializationHelper = new PrimitiveDeserializationHelper(mySchema = schema);
|
||||
myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length) / 8)));
|
||||
myInitReorderSchema();
|
||||
|
||||
|
||||
}
|
||||
|
||||
#region Schema Reordering initialization
|
||||
private void myInitReorderSchema ()
|
||||
{
|
||||
int idColIndex = mySchema["id"].colNo;
|
||||
int[] temp = new int [mySchema.columnInfo.Length];
|
||||
foreach (ColumnInfo ci in mySchema.columnInfo)
|
||||
{
|
||||
temp[ci.colNo] =
|
||||
ci.colNo == idColIndex ? 0 :
|
||||
ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo;
|
||||
private void myInitReorderSchema() {
|
||||
try {
|
||||
int idColIndex = mySchema["id"].colNo;
|
||||
int[] temp = new int[mySchema.columnInfo.Length];
|
||||
foreach (ColumnInfo ci in mySchema.columnInfo) {
|
||||
temp[ci.colNo] =
|
||||
ci.colNo == idColIndex ? 0 :
|
||||
ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo;
|
||||
}
|
||||
mySerIndexToSchemaIndexMap = new int[temp.Length];
|
||||
for (int i = 0; i < temp.Length; i++)
|
||||
mySerIndexToSchemaIndexMap[temp[i]] = i;
|
||||
} catch (Exception ex) {
|
||||
string schema_name = mySchema == null ? "NULL" : mySchema.tableName;
|
||||
string err_msg = $"ClientUniversalRowDeserializer.myInitReorderSchema table: {schema_name} ex: {ex}";
|
||||
bts.Log.Gen.error(err_msg);
|
||||
throw new ApplicationException(err_msg);
|
||||
}
|
||||
mySerIndexToSchemaIndexMap = new int[temp.Length];
|
||||
for (int i = 0; i < temp.Length; i ++)
|
||||
mySerIndexToSchemaIndexMap [temp[i]] = i;
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region IClientRowDeserializer Members
|
||||
|
||||
public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
|
||||
{
|
||||
br.ReadInt32 (); //length information for the row not needed
|
||||
myNullMap = new BitArray(br.ReadBytes (myNullMapLength));
|
||||
object[] values = new object [mySchema.columnNames.Length];
|
||||
for (int i = 0; i < values.Length; i++)
|
||||
{
|
||||
public void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) {
|
||||
br.ReadInt32(); //length information for the row not needed
|
||||
myNullMap = new BitArray(br.ReadBytes(myNullMapLength));
|
||||
object[] values = new object[mySchema.columnNames.Length];
|
||||
for (int i = 0; i < values.Length; i++) {
|
||||
int colNo = mySerIndexToSchemaIndexMap[i];
|
||||
values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn (colNo, br);
|
||||
values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn(colNo, br);
|
||||
}
|
||||
|
||||
|
||||
r = new BlueRow(mySchema, values);
|
||||
r = new BlueRow(mySchema, values);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -1,22 +1,149 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
|
||||
namespace BCG {
|
||||
public class StreamIOMessage {
|
||||
public enum StreamOp {
|
||||
Create = 0,
|
||||
Add = 1, //adds a single row
|
||||
Remove = 2, //removes a single row
|
||||
Change = 3, //changes a single row
|
||||
MultipleRemoveAdd = 4,//removes and adds multiple rows
|
||||
MultipleChange = 5 //changes multiples rows
|
||||
}
|
||||
public class CreateMessage {
|
||||
public IBlueRowCollection rows;
|
||||
public override string ToString() {
|
||||
string rv = "Create message ";
|
||||
if (rows == null) {
|
||||
rv += " rows: null";
|
||||
return rv;
|
||||
}
|
||||
foreach (IBlueRow r in rows) {
|
||||
rv += $"id: {r["id"]} ";
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
public class AddMessage {
|
||||
public IBlueRow added_row;
|
||||
}
|
||||
public class RemoveMessage {
|
||||
public IBlueRow removed_row;
|
||||
}
|
||||
public class ChangeMessage {
|
||||
public IBlueRow changed_row;
|
||||
}
|
||||
public class MultipleRemoveAddMessage {
|
||||
public IBlueRowCollection removed;
|
||||
public IBlueRowCollection added;
|
||||
public override string ToString() {
|
||||
string rv = "MultipleRemoveAdd removed: ";
|
||||
foreach (IBlueRow r in removed) {
|
||||
rv += $"id: {r["id"]} ";
|
||||
}
|
||||
rv += "added: ";
|
||||
foreach (IBlueRow r in added) {
|
||||
rv += $"id: {r["id"]} ";
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
public class MultipleChangeMessage {
|
||||
public IBlueRowCollection changed;
|
||||
}
|
||||
|
||||
public StreamOp op;
|
||||
public CreateMessage create_message = null;
|
||||
public AddMessage add_message = null;
|
||||
public RemoveMessage remove_message = null;
|
||||
public ChangeMessage change_message = null;
|
||||
public MultipleRemoveAddMessage multiple_remove_add_message = null;
|
||||
public MultipleChangeMessage multiple_change_message = null;
|
||||
public StreamIOMessage(byte op_, object data) {
|
||||
op = (StreamIOMessage.StreamOp)(op_);
|
||||
switch (op) {
|
||||
case StreamIOMessage.StreamOp.Create:
|
||||
IBlueRowCollection state = null;
|
||||
// TODO, not sure why it can be an array and by itself
|
||||
if (data is IBlueRowCollection) state = data as IBlueRowCollection;
|
||||
else if (data is IBlueRowCollection[]) state = (data as IBlueRowCollection[]).ElementAt(0);
|
||||
create_message = new CreateMessage() { rows = state };
|
||||
break;
|
||||
case StreamIOMessage.StreamOp.Add:
|
||||
bts.Log.assert(false);
|
||||
break;
|
||||
case StreamIOMessage.StreamOp.Remove:
|
||||
bts.Log.assert(false);
|
||||
break;
|
||||
case StreamIOMessage.StreamOp.Change:
|
||||
bts.Log.assert(false);
|
||||
break;
|
||||
case StreamIOMessage.StreamOp.MultipleRemoveAdd:
|
||||
IBlueRowCollection[] row_args = data as IBlueRowCollection[];
|
||||
multiple_remove_add_message = new MultipleRemoveAddMessage() { removed = row_args[0], added = row_args[1] };
|
||||
break;
|
||||
case StreamIOMessage.StreamOp.MultipleChange:
|
||||
//IBlueRowCollection[] row_args = data as IBlueRowCollection[];
|
||||
multiple_change_message = new MultipleChangeMessage() { changed = (data as IBlueRowCollection[])[0] };
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public override string ToString() {
|
||||
switch (op) {
|
||||
case StreamIOMessage.StreamOp.Create:
|
||||
return create_message.ToString();
|
||||
break;
|
||||
case StreamIOMessage.StreamOp.Add:
|
||||
bts.Log.assert(false);
|
||||
return String.Empty;
|
||||
break;
|
||||
case StreamIOMessage.StreamOp.Remove:
|
||||
bts.Log.assert(false);
|
||||
return String.Empty;
|
||||
break;
|
||||
case StreamIOMessage.StreamOp.Change:
|
||||
bts.Log.assert(false);
|
||||
return String.Empty;
|
||||
break;
|
||||
case StreamIOMessage.StreamOp.MultipleRemoveAdd:
|
||||
return multiple_remove_add_message.ToString();
|
||||
break;
|
||||
case StreamIOMessage.StreamOp.MultipleChange:
|
||||
return multiple_change_message.ToString();
|
||||
break;
|
||||
}
|
||||
bts.Log.assert(false);
|
||||
return String.Empty;
|
||||
}
|
||||
public static IEnumerable<StreamIOMessage> assemble_msgs(byte[] ops, Array op_data) {
|
||||
int i = 0;
|
||||
List<StreamIOMessage> rv = new List<StreamIOMessage>();
|
||||
foreach (byte op in ops) {
|
||||
object data = op_data.GetValue(i++);
|
||||
rv.Add(new StreamIOMessage(op, data));
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
/// <summary>
|
||||
/// Performs deserialization inside the client app. The client gets either the complete data set or the
|
||||
/// increments from PubSub and this interface makes sense out of them from a tabular/datastream stand point.
|
||||
/// </summary>
|
||||
public interface IClientDeserializer
|
||||
{
|
||||
public interface IClientDeserializer {
|
||||
/// <summary>
|
||||
/// deserializes the data received by the client from the pubsub
|
||||
/// </summary>
|
||||
/// <param name="serializedData">the data in byte format received from pubsub</param>
|
||||
/// <param name="theOps">the operations that the data maps onto</param>
|
||||
/// <param name="theRows">the data corresponding to the operations, is an array of IblueRowCollection[]</param>
|
||||
void deserialize (Stream[] streams, out byte[] theOps, out Array theRows);
|
||||
void deserialize(Stream[] streams, out byte[] theOps, out Array theRows);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
+3
-5
@@ -2,13 +2,11 @@ using System;
|
||||
using System.IO;
|
||||
using System.Collections;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// Deserializes one row. the current state is also provided as some deserialization schemes might use it.
|
||||
/// </summary>
|
||||
public interface IClientRowDeserializer
|
||||
{
|
||||
void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r);
|
||||
public interface IClientRowDeserializer {
|
||||
void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r);
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,7 @@ namespace BCG {
|
||||
|
||||
public PubTablePublisher(string tag, PublisherCore publisherCore) {
|
||||
_tag = tag;
|
||||
bts.Log.Gen.info($"PubTablePublisher created for tag: {tag}");
|
||||
bts.Log.Gen.info($"PubTablePublisher.cons created for tag: {tag}");
|
||||
myPubCore = publisherCore;
|
||||
mySerializer = new BlueDataSerializationController();
|
||||
}
|
||||
@@ -54,10 +54,14 @@ namespace BCG {
|
||||
out data);
|
||||
bts.Log.Gen.info($"PubTablePublisher.createTable tag: {_tag} rows: {rows.Count} data: {data.Length}");
|
||||
myPublish(data, rows.schema.tableName);
|
||||
// byte[] ops;
|
||||
// Array rows_ds;
|
||||
// _test_deserializer.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows_ds);
|
||||
}
|
||||
|
||||
//private ClientDeserializationController _test_deserializer = new ClientDeserializationController();
|
||||
public virtual void batchRemoveAdd(IBlueRowCollection delRows, IBlueRowCollection addRows) {
|
||||
byte[] data;
|
||||
byte[] data = null;
|
||||
if (myIsRealChange(delRows, addRows))
|
||||
batchChange(addRows);
|
||||
else {
|
||||
@@ -66,6 +70,9 @@ namespace BCG {
|
||||
out data);
|
||||
myPublish(data, delRows.schema.tableName);
|
||||
}
|
||||
// byte[] ops;
|
||||
// Array rows_ds;
|
||||
// _test_deserializer.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows_ds);
|
||||
}
|
||||
|
||||
public virtual void batchChange(IBlueRowCollection rows) {
|
||||
|
||||
@@ -1,40 +1,84 @@
|
||||
using System;
|
||||
using mbts.config;
|
||||
using bts.utils;
|
||||
using System.Threading.Tasks;
|
||||
using Bts.DataStreamExpressions;
|
||||
using System.Collections.Generic;
|
||||
|
||||
using BCG;
|
||||
|
||||
bts.utils.BTSNativeMethods.provide_command_line(args, args.Length);
|
||||
mbts.config.Configuration.initialize();
|
||||
|
||||
bts.Log.Gen.info($"Starting DSPublisher");
|
||||
Console.WriteLine("Starting DSPublisher...");
|
||||
// static async Task _run() {
|
||||
// var login = "bts";
|
||||
// var passwd = String.Empty;
|
||||
// Console.WriteLine("Starting DSQA... ");
|
||||
// // Log.notnull(login);
|
||||
// // Log.notnull(passwd);
|
||||
// // var service_ctx = bts.api.core.ServiceContext.create();
|
||||
// await service_ctx.login(login, passwd);
|
||||
// }
|
||||
//Task.Run(async () => await _run());
|
||||
|
||||
_publish_stream();
|
||||
|
||||
void _publish_stream() {
|
||||
|
||||
BCG.ViewStream vs = new LiteralStream("x", "420.0");// new BBOStream();
|
||||
//new BCG.MDStream();
|
||||
ViewPublisher vpub = new ViewPublisher(String.Empty, new PubTableBroker(), vs);
|
||||
vs.onChange += on_change;
|
||||
static void init(string[] args)
|
||||
{
|
||||
Console.WriteLine("Hit key to continue");
|
||||
Console.ReadLine();
|
||||
Func<string, string, Task> login = async (l, p) =>
|
||||
{
|
||||
var login = l;
|
||||
var passwd = p;
|
||||
var service_ctx = bts.api.core.ServiceContext.create();
|
||||
await service_ctx.login(login, passwd);
|
||||
};
|
||||
BTSNativeMethods.provide_command_line(args, args.Length);
|
||||
Configuration.initialize();
|
||||
Task.Run(async () => await login("bts", String.Empty));
|
||||
}
|
||||
while (true) {
|
||||
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
|
||||
ViewStreamDatabase.instance.doWork();
|
||||
|
||||
init(args);
|
||||
_publish_streams();
|
||||
|
||||
static ViewStream build_stream(string stream_expression)
|
||||
{
|
||||
try
|
||||
{
|
||||
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
|
||||
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||
return rv;
|
||||
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void on_change(object obj) {
|
||||
DataPacket dp = obj as DataPacket;
|
||||
dp.newRows.print_table((col) => { return 10; });
|
||||
void _publish_streams()
|
||||
{
|
||||
PubTableBroker pb_broker = new PubTableBroker();
|
||||
CloudStreamManager csm = new CloudStreamManager();
|
||||
// get it from config
|
||||
Dictionary<string, string> streams_to_publish = new Dictionary<string, string>() {
|
||||
{"bbo","bbo()"},
|
||||
{"instruments","instruments()"},
|
||||
//{"timesales","timesales(10000)"},
|
||||
{"books","books()"},
|
||||
// {"seconds_1_10","second(1,10)"},
|
||||
{"literal_420","literal(\"x\", \"420.0\")"},
|
||||
// {"bbo_ts","timeSeries(bbo())"}
|
||||
};
|
||||
foreach (var kv in streams_to_publish)
|
||||
{
|
||||
var name = kv.Key;
|
||||
var exp = kv.Value;
|
||||
BCG.ViewStream vs = build_stream(exp);
|
||||
ViewPublisher vpub = new ViewPublisher(name, pb_broker, vs);
|
||||
vs.onChange += on_change;
|
||||
}
|
||||
while (true)
|
||||
{
|
||||
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
|
||||
ViewStreamDatabase.instance.doWork();
|
||||
csm.doWork();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
void on_change(object obj)
|
||||
{
|
||||
bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
|
||||
}
|
||||
@@ -25,17 +25,19 @@ using System.Threading.Tasks;
|
||||
|
||||
|
||||
|
||||
namespace BCG {
|
||||
public partial class MainForm : System.Windows.Forms.Form {
|
||||
static async Task _run() {
|
||||
var login = "bts";
|
||||
var passwd = String.Empty;
|
||||
Console.WriteLine("Starting DSQA... ");
|
||||
// Log.notnull(login);
|
||||
// Log.notnull(passwd);
|
||||
var service_ctx = bts.api.core.ServiceContext.create();
|
||||
await service_ctx.login(login, passwd);
|
||||
}
|
||||
namespace BCG
|
||||
{
|
||||
public partial class MainForm : System.Windows.Forms.Form
|
||||
{
|
||||
// static async Task _run() {
|
||||
// var login = "bts";
|
||||
// var passwd = String.Empty;
|
||||
// Console.WriteLine("Starting DSQA... ");
|
||||
// // Log.notnull(login);
|
||||
// // Log.notnull(passwd);
|
||||
// var service_ctx = bts.api.core.ServiceContext.create();
|
||||
// await service_ctx.login(login, passwd);
|
||||
// }
|
||||
private bool myChangeStyle = false;
|
||||
private PubSubRoute[] myRoutes = PubSub3Config.instance.getSubRouting();
|
||||
private PersistWindowState myWindowState;
|
||||
@@ -50,8 +52,9 @@ namespace BCG {
|
||||
private string myKQ;
|
||||
private TextWriter myConsoleOutput;
|
||||
|
||||
public MainForm() {
|
||||
Task.Run(async () => await _run());
|
||||
public MainForm()
|
||||
{
|
||||
//Task.Run(async () => await _run());
|
||||
InitializeComponent();
|
||||
// myKQ = ConfigurationRequester.getApplicationQualifier();
|
||||
myKQ = "TODO";
|
||||
@@ -65,17 +68,20 @@ namespace BCG {
|
||||
this.Closing += new CancelEventHandler(myClosing);
|
||||
}
|
||||
|
||||
public string query_text {
|
||||
public string query_text
|
||||
{
|
||||
get { return myQueryTextBox.Text; }
|
||||
set { myQueryTextBox.Text = value; }
|
||||
}
|
||||
private ViewStream myGetAllTagsStream() {
|
||||
private ViewStream myGetAllTagsStream()
|
||||
{
|
||||
// TODO: Port PublishedTagsStream from edge (then we can uncomment impl below)
|
||||
return new FilterStream(new PubSubServerStateStream(), "tag~='DS_'");
|
||||
//return new LiteralStream("tag/String", "DS_Comedians;DS_Baldies");
|
||||
}
|
||||
|
||||
private void mySetText() {
|
||||
private void mySetText()
|
||||
{
|
||||
this.Text = "Data Stream Query Analyzer | Site: " + mySite + (string.IsNullOrEmpty(myKQ) ? string.Empty : " | Key Qualifier: " + myKQ);
|
||||
myToolStripStatusLabelTime.Text = "Ready";
|
||||
myToolStripStatusLabelRows.Text = string.Empty;
|
||||
@@ -85,10 +91,12 @@ namespace BCG {
|
||||
/// Gets the data in the form of a multilevel IDictionary
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
private IDictionary myGetLibraryData() {
|
||||
private IDictionary myGetLibraryData()
|
||||
{
|
||||
SortedList retVal = new SortedList();
|
||||
string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames();
|
||||
foreach (string pName in primitiveNames) {
|
||||
foreach (string pName in primitiveNames)
|
||||
{
|
||||
string aName = PrimitiveManager.instance.getAssemblyName(pName);
|
||||
ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName);
|
||||
if (!retVal.Contains(aName))
|
||||
@@ -103,12 +111,15 @@ namespace BCG {
|
||||
}
|
||||
|
||||
|
||||
private void myLoadAssemblies() {
|
||||
private void myLoadAssemblies()
|
||||
{
|
||||
myTreeView.Nodes.Clear();
|
||||
IDictionary libraryData = myGetLibraryData();
|
||||
foreach (DictionaryEntry de in libraryData) {
|
||||
foreach (DictionaryEntry de in libraryData)
|
||||
{
|
||||
TreeNode assemblyNode = new TreeNode(de.Key as String);
|
||||
foreach (DictionaryEntry innerDe in (de.Value as IDictionary)) {
|
||||
foreach (DictionaryEntry innerDe in (de.Value as IDictionary))
|
||||
{
|
||||
TreeNode primitiveNode = new TreeNode(innerDe.Key as string);
|
||||
foreach (string con in (innerDe.Value as ICollection))
|
||||
primitiveNode.Nodes.Add(con);
|
||||
@@ -128,23 +139,27 @@ namespace BCG {
|
||||
}*/
|
||||
}
|
||||
|
||||
private void myHandleTableListChange(object o) {
|
||||
private void myHandleTableListChange(object o)
|
||||
{
|
||||
IBlueRowCollection rows = (o as DataPacket).newRows;
|
||||
myUpdateTableList();
|
||||
}
|
||||
|
||||
|
||||
#region Persistence Keys
|
||||
public int textBoxHeight {
|
||||
public int textBoxHeight
|
||||
{
|
||||
get { return myQueryTextBox.Height; }
|
||||
set { myQueryTextBox.Height = value; }
|
||||
}
|
||||
#endregion
|
||||
|
||||
private void myTryExpression(string expStr) {
|
||||
private void myTryExpression(string expStr)
|
||||
{
|
||||
myFreezeView(false); // force unfreeze the view
|
||||
DataStreamExpression oldExp = myExp;
|
||||
try {
|
||||
try
|
||||
{
|
||||
var exp = DataStreamExpressionsInterpreter.Evaluate(expStr);
|
||||
|
||||
if (exp == null)
|
||||
@@ -171,7 +186,9 @@ namespace BCG {
|
||||
oldExp.getView(new Hashtable()).onChange -= new DataStream.ChangeDelegate(myCallback);
|
||||
*/
|
||||
//myQueryTextBox.Text = expStr;
|
||||
} catch (Exception exc) {
|
||||
}
|
||||
catch (Exception exc)
|
||||
{
|
||||
Log.Ui.warn(exc.ToString());
|
||||
|
||||
myDataGrid.DataSource = null;
|
||||
@@ -185,7 +202,8 @@ namespace BCG {
|
||||
}
|
||||
}
|
||||
|
||||
private string myExtractFromClass(TextReader r) {
|
||||
private string myExtractFromClass(TextReader r)
|
||||
{
|
||||
string line = r.ReadLine();
|
||||
while (line != null && line.IndexOf("*Expression*") == -1)
|
||||
line = r.ReadLine();
|
||||
@@ -198,7 +216,8 @@ namespace BCG {
|
||||
string result = "";
|
||||
line = r.ReadLine();
|
||||
|
||||
while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1) {
|
||||
while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1)
|
||||
{
|
||||
if (line != "")
|
||||
result += line.Substring(" ".Length) + "\n";
|
||||
line = r.ReadLine();
|
||||
@@ -209,17 +228,24 @@ namespace BCG {
|
||||
return result;
|
||||
}
|
||||
|
||||
public void loadFile(string fileName) {
|
||||
try {
|
||||
public void loadFile(string fileName)
|
||||
{
|
||||
try
|
||||
{
|
||||
TextReader r = new StreamReader(fileName);
|
||||
try {
|
||||
try
|
||||
{
|
||||
myQueryTextBox.Text = myExtractFromClass(r);
|
||||
} finally {
|
||||
}
|
||||
finally
|
||||
{
|
||||
r.Close();
|
||||
}
|
||||
myLastFileName = fileName;
|
||||
this.Text = fileName;
|
||||
} catch (Exception exc) {
|
||||
}
|
||||
catch (Exception exc)
|
||||
{
|
||||
Log.Ui.warn(exc.ToString());
|
||||
|
||||
myDataGrid.DataSource = null;
|
||||
@@ -230,7 +256,8 @@ namespace BCG {
|
||||
|
||||
}
|
||||
|
||||
private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code) {
|
||||
private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code)
|
||||
{
|
||||
|
||||
myCSharpText.Text = "using System;\n\nnamespace BCG\n{";
|
||||
|
||||
@@ -250,7 +277,8 @@ namespace BCG {
|
||||
|
||||
string lastVar = null;
|
||||
string variableDeclarations = "";
|
||||
foreach (string var in vars) {
|
||||
foreach (string var in vars)
|
||||
{
|
||||
variableDeclarations += " ViewStream " + var + ";\n";
|
||||
lastVar = var;
|
||||
}
|
||||
@@ -280,13 +308,16 @@ namespace BCG {
|
||||
myCSharpText.Text += "}";
|
||||
}
|
||||
|
||||
static DataGridViewColumn _column_factory(ColumnInfo ci) {
|
||||
return new DataGridViewTextBoxColumn {
|
||||
static DataGridViewColumn _column_factory(ColumnInfo ci)
|
||||
{
|
||||
return new DataGridViewTextBoxColumn
|
||||
{
|
||||
HeaderText = ci.colName,
|
||||
Width = get_col_width(ci)
|
||||
};
|
||||
|
||||
static int get_col_width(ColumnInfo ci) {
|
||||
static int get_col_width(ColumnInfo ci)
|
||||
{
|
||||
const int CHAR_WIDTH = 10;
|
||||
const int MIN_WIDTH = 50;
|
||||
const int MIN_DATE_WIDTH = 200;
|
||||
@@ -297,12 +328,14 @@ namespace BCG {
|
||||
}
|
||||
private HighPerformanceTimer _timer = new HighPerformanceTimer();
|
||||
// this is the stream callback
|
||||
private void myCallback(object o) {
|
||||
private void myCallback(object o)
|
||||
{
|
||||
_timer.start();
|
||||
DataPacket dp = o as DataPacket;
|
||||
myCrono.stop();
|
||||
Log.Ui.diag($"Received callback for {dp.schema.tableName}");
|
||||
if (o == null) {
|
||||
if (o == null)
|
||||
{
|
||||
System.Windows.Forms.MessageBox.Show(this, "Received a Null Data packet",
|
||||
"Invalid Data", MessageBoxButtons.OK, MessageBoxIcon.Error);
|
||||
return;
|
||||
@@ -319,7 +352,8 @@ namespace BCG {
|
||||
|
||||
myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows);
|
||||
var force = false;
|
||||
if (myChangeStyle && force) {
|
||||
if (myChangeStyle && force)
|
||||
{
|
||||
myChangeStyle = false;
|
||||
myDataGrid.Columns.Clear();
|
||||
myDataGrid.Columns.AddRange(Array.ConvertAll(dp.schema.columnInfo, _column_factory));
|
||||
@@ -346,11 +380,15 @@ namespace BCG {
|
||||
|
||||
private DataStreamExpression myExp;
|
||||
|
||||
private void myTimer_Tick(object sender, System.EventArgs e) {
|
||||
try {
|
||||
private void myTimer_Tick(object sender, System.EventArgs e)
|
||||
{
|
||||
try
|
||||
{
|
||||
ViewStreamDatabase.instance.doWork();
|
||||
MasterWorker.getInstance(false).doWork();
|
||||
} catch (Exception exc) {
|
||||
}
|
||||
catch (Exception exc)
|
||||
{
|
||||
Log.Ui.warn(exc.ToString());
|
||||
|
||||
myDataGrid.DataSource = null;
|
||||
@@ -361,22 +399,27 @@ namespace BCG {
|
||||
}
|
||||
}
|
||||
|
||||
class StringWrapper {
|
||||
public StringWrapper(string s) {
|
||||
class StringWrapper
|
||||
{
|
||||
public StringWrapper(string s)
|
||||
{
|
||||
myTableName = s;
|
||||
}
|
||||
public string tableName {
|
||||
public string tableName
|
||||
{
|
||||
get { return myTableName; }
|
||||
}
|
||||
private string myTableName;
|
||||
}
|
||||
|
||||
private void MainForm_Closed(object sender, System.EventArgs e) {
|
||||
private void MainForm_Closed(object sender, System.EventArgs e)
|
||||
{
|
||||
if (myExp != null)
|
||||
myExp.getView(new Dictionary<string, StatementExpression>(0)).onChange -= new DataStream.ChangeDelegate(myCallback);
|
||||
}
|
||||
|
||||
private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) {
|
||||
private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e)
|
||||
{
|
||||
myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
|
||||
myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
|
||||
myQueryTextBox.SelectionColor = Color.Black;
|
||||
@@ -384,22 +427,24 @@ namespace BCG {
|
||||
myTryExpression(myQueryTextBox.Text);
|
||||
}
|
||||
|
||||
private void myToolStripMenuItemExecSel_Click(object sender, EventArgs e) {
|
||||
int selstart = myQueryTextBox.SelectionStart;
|
||||
int sellength = myQueryTextBox.SelectionLength;
|
||||
myQueryTextBox.SelectionStart = 0;
|
||||
myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
|
||||
myQueryTextBox.SelectionColor = Color.Black;
|
||||
myQueryTextBox.SelectionStart = selstart;
|
||||
myQueryTextBox.SelectionLength = sellength;
|
||||
myQueryTextBox.SelectionColor = Color.Crimson;
|
||||
myTryExpression(myQueryTextBox.SelectedText);
|
||||
myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
|
||||
myQueryTextBox.SelectionLength = 0;
|
||||
myQueryTextBox.SelectionColor = Color.Black;
|
||||
private void myToolStripMenuItemExecSel_Click(object sender, EventArgs e)
|
||||
{
|
||||
// int selstart = myQueryTextBox.SelectionStart;
|
||||
// int sellength = myQueryTextBox.SelectionLength;
|
||||
// myQueryTextBox.SelectionStart = 0;
|
||||
// myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
|
||||
// myQueryTextBox.SelectionColor = Color.White;
|
||||
// myQueryTextBox.SelectionStart = selstart;
|
||||
// myQueryTextBox.SelectionLength = sellength;
|
||||
// myQueryTextBox.SelectionColor = Color.Crimson;
|
||||
// myTryExpression(myQueryTextBox.SelectedText);
|
||||
// myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
|
||||
// myQueryTextBox.SelectionLength = 0;
|
||||
// myQueryTextBox.SelectionColor = Color.White;
|
||||
}
|
||||
|
||||
private void myMenuItemOpenClick(object sender, System.EventArgs e) {
|
||||
private void myMenuItemOpenClick(object sender, System.EventArgs e)
|
||||
{
|
||||
OpenFileDialog od = new OpenFileDialog();
|
||||
od.Filter = "C# class (*.cs)|*.cs";
|
||||
|
||||
@@ -407,21 +452,24 @@ namespace BCG {
|
||||
loadFile(od.FileName);
|
||||
}
|
||||
|
||||
private void mySave(string fileName) {
|
||||
private void mySave(string fileName)
|
||||
{
|
||||
using (TextWriter w = new StreamWriter(fileName))
|
||||
w.Write(myCSharpText.Text);
|
||||
myLastFileName = fileName;
|
||||
this.Text = fileName;
|
||||
}
|
||||
|
||||
private void myMenuItemSaveClick(object sender, System.EventArgs e) {
|
||||
private void myMenuItemSaveClick(object sender, System.EventArgs e)
|
||||
{
|
||||
if (myLastFileName == null)
|
||||
myMenuItemSaveAsClick(sender, e);
|
||||
else
|
||||
mySave(myLastFileName);
|
||||
}
|
||||
|
||||
private void myMenuItemSaveAsClick(object sender, System.EventArgs e) {
|
||||
private void myMenuItemSaveAsClick(object sender, System.EventArgs e)
|
||||
{
|
||||
SaveFileDialog sd = new SaveFileDialog();
|
||||
if (myLastFileName != null)
|
||||
sd.FileName = myLastFileName;
|
||||
@@ -432,47 +480,56 @@ namespace BCG {
|
||||
mySave(sd.FileName);
|
||||
}
|
||||
|
||||
private void myTableListBox_DoubleClick(object sender, System.EventArgs e) {
|
||||
if (myTableListBox.SelectedItem != null) {
|
||||
private void myTableListBox_DoubleClick(object sender, System.EventArgs e)
|
||||
{
|
||||
if (myTableListBox.SelectedItem != null)
|
||||
{
|
||||
string tableName = myTableListBox.SelectedItem.ToString();
|
||||
myQueryTextBox.Text = myQueryTextBox.Text == "" ? tableName : myQueryTextBox.Text + "\n" + tableName;
|
||||
myTryExpression(myQueryTextBox.Text);
|
||||
}
|
||||
}
|
||||
|
||||
private void myDataGrid_MouseEnter(object sender, System.EventArgs e) {
|
||||
private void myDataGrid_MouseEnter(object sender, System.EventArgs e)
|
||||
{
|
||||
//myDataGrid.TableStyles[0].GridLineColor = Color.LightSteelBlue;
|
||||
// if (myDataGrid.TableStyles.Count > 0)
|
||||
// myDataGrid.TableStyles[0].HeaderBackColor = Color.LightSteelBlue;
|
||||
}
|
||||
|
||||
private void myDataGrid_MouseLeave(object sender, System.EventArgs e) {
|
||||
private void myDataGrid_MouseLeave(object sender, System.EventArgs e)
|
||||
{
|
||||
//myDataGrid.TableStyles[0].GridLineColor = Color.Gainsboro;
|
||||
// if (myDataGrid.TableStyles.Count > 0)
|
||||
// myDataGrid.TableStyles[0].HeaderBackColor = Color.Gainsboro;
|
||||
}
|
||||
|
||||
private void myCSharpText_MouseEnter(object sender, System.EventArgs e) {
|
||||
private void myCSharpText_MouseEnter(object sender, System.EventArgs e)
|
||||
{
|
||||
//myCSharpText.BackColor = Color.Gainsboro;
|
||||
myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Bold);
|
||||
}
|
||||
|
||||
private void myCSharpText_MouseLeave(object sender, System.EventArgs e) {
|
||||
private void myCSharpText_MouseLeave(object sender, System.EventArgs e)
|
||||
{
|
||||
//myCSharpText.BackColor = Color.White;
|
||||
myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Regular);
|
||||
}
|
||||
|
||||
private void myTableListBox_MouseEnter(object sender, System.EventArgs e) {
|
||||
private void myTableListBox_MouseEnter(object sender, System.EventArgs e)
|
||||
{
|
||||
//myTableListBox.BackColor = Color.Gainsboro;
|
||||
myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Bold);
|
||||
}
|
||||
|
||||
private void myTableListBox_MouseLeave(object sender, System.EventArgs e) {
|
||||
private void myTableListBox_MouseLeave(object sender, System.EventArgs e)
|
||||
{
|
||||
//myTableListBox.BackColor = Color.White;
|
||||
myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Regular);
|
||||
}
|
||||
|
||||
private void myLoad(object sender, EventArgs e) {
|
||||
private void myLoad(object sender, EventArgs e)
|
||||
{
|
||||
myFreezeView(false);
|
||||
|
||||
myToolStripMenuItemSite.DropDownItems.Clear();
|
||||
@@ -496,7 +553,9 @@ namespace BCG {
|
||||
{
|
||||
var ex = new Win32Exception(); // same as new Win32Exception(Marshal.GetLastWin32Error())
|
||||
Console.WriteLine(ex.Message);
|
||||
} else {
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine("Set stdout worked.");
|
||||
}
|
||||
|
||||
@@ -504,7 +563,9 @@ namespace BCG {
|
||||
{
|
||||
var ex = new Win32Exception(); // same as new Win32Exception(Marshal.GetLastWin32Error())
|
||||
Console.WriteLine(ex.Message);
|
||||
} else {
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine("Set stderr worked.");
|
||||
}
|
||||
Console.WriteLine("Redirecting console ouput to this window...");
|
||||
@@ -520,10 +581,12 @@ namespace BCG {
|
||||
//_library = new LibraryForm(this);
|
||||
}
|
||||
|
||||
private void myMenuItemSiteClick(object sender, System.EventArgs e) {
|
||||
private void myMenuItemSiteClick(object sender, System.EventArgs e)
|
||||
{
|
||||
ToolStripMenuItem mi = (ToolStripMenuItem)sender;
|
||||
|
||||
if (mi.Text != mySite) {
|
||||
if (mi.Text != mySite)
|
||||
{
|
||||
mySite = mi.Text;
|
||||
foreach (var m in mySiteItems)
|
||||
m.Checked = false;
|
||||
@@ -544,63 +607,78 @@ namespace BCG {
|
||||
}
|
||||
}
|
||||
|
||||
private void myClosing(object sender, CancelEventArgs e) {
|
||||
private void myClosing(object sender, CancelEventArgs e)
|
||||
{
|
||||
}
|
||||
|
||||
private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e) {
|
||||
if (myTreeView.SelectedNode != null) {
|
||||
if (myTreeView.SelectedNode.LastNode == null) {
|
||||
private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e)
|
||||
{
|
||||
if (myTreeView.SelectedNode != null)
|
||||
{
|
||||
if (myTreeView.SelectedNode.LastNode == null)
|
||||
{
|
||||
string sn = myTreeView.SelectedNode.Parent.Text + "()";
|
||||
myQueryTextBox.Text = myQueryTextBox.Text == "" ? sn : myQueryTextBox.Text + "\n" + sn;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void myToolStripMenuItemExit_Click(object sender, EventArgs e) {
|
||||
private void myToolStripMenuItemExit_Click(object sender, EventArgs e)
|
||||
{
|
||||
Close();
|
||||
}
|
||||
|
||||
private void myToolStripMenuItemFreeze_Click(object sender, EventArgs e) {
|
||||
private void myToolStripMenuItemFreeze_Click(object sender, EventArgs e)
|
||||
{
|
||||
myFreezeView(myTimer.Enabled);
|
||||
}
|
||||
|
||||
private void myFreezeView(bool freeze) {
|
||||
private void myFreezeView(bool freeze)
|
||||
{
|
||||
myTimer.Enabled = !freeze;
|
||||
myToolStripMenuItemFreeze.Checked = freeze;
|
||||
myToolStripMenuItemStep.Enabled = freeze;
|
||||
}
|
||||
|
||||
private void myToolStripMenuItemStep_Click(object sender, EventArgs e) {
|
||||
private void myToolStripMenuItemStep_Click(object sender, EventArgs e)
|
||||
{
|
||||
ViewStreamDatabase.instance.doWork();
|
||||
}
|
||||
|
||||
private void myToolStripMenuItemLoadAss_Click(object sender, EventArgs e) {
|
||||
private void myToolStripMenuItemLoadAss_Click(object sender, EventArgs e)
|
||||
{
|
||||
OpenFileDialog ofd = new OpenFileDialog();
|
||||
ofd.ShowReadOnly = true;
|
||||
DialogResult dr = ofd.ShowDialog(this);
|
||||
if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK) {
|
||||
if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK)
|
||||
{
|
||||
PrimitiveManager.instance.loadAssembly(ofd.FileName);
|
||||
myLoadAssemblies();
|
||||
}
|
||||
}
|
||||
|
||||
private void myToolStripMenuItemTest_Click(object sender, EventArgs e) {
|
||||
if (myCurrentStream != null) {
|
||||
private void myToolStripMenuItemTest_Click(object sender, EventArgs e)
|
||||
{
|
||||
if (myCurrentStream != null)
|
||||
{
|
||||
myCurrentStream.onChange -= new DataStream.ChangeDelegate(myCallback);
|
||||
myCurrentStream.onChange += new DataStream.ChangeDelegate(myCallback);
|
||||
}
|
||||
}
|
||||
|
||||
private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e) {
|
||||
private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e)
|
||||
{
|
||||
myConsoleRichTextBox.Clear();
|
||||
Console.WriteLine("Redirecting console ouput to this window...");
|
||||
}
|
||||
|
||||
private void myTableFilterBox_TextChanged(object sender, EventArgs e) {
|
||||
private void myTableFilterBox_TextChanged(object sender, EventArgs e)
|
||||
{
|
||||
myUpdateTableList();
|
||||
}
|
||||
|
||||
private void myUpdateTableList() {
|
||||
private void myUpdateTableList()
|
||||
{
|
||||
IBlueRowCollection rows = myAllTagsStream.rows;
|
||||
if (rows == null)
|
||||
return;
|
||||
@@ -615,12 +693,15 @@ namespace BCG {
|
||||
myTableListBox.Items.Clear();
|
||||
SortedList<string, bool> sl = new SortedList<string, bool>();
|
||||
if (string.IsNullOrEmpty(myTableFilterBox.Text))
|
||||
foreach (IBlueRow r in rows) {
|
||||
foreach (IBlueRow r in rows)
|
||||
{
|
||||
sl[r["tag"].ToString().Substring(PubStreamPrefixer.prefix("").Length)] = true;
|
||||
}
|
||||
else {
|
||||
else
|
||||
{
|
||||
Regex tableRegex = new Regex(myTableFilterBox.Text);
|
||||
foreach (IBlueRow r in rows) {
|
||||
foreach (IBlueRow r in rows)
|
||||
{
|
||||
string thisTable = r["tag"].ToString().Substring(PubStreamPrefixer.prefix("").Length);
|
||||
if (tableRegex.IsMatch(thisTable))
|
||||
sl[thisTable] = true;
|
||||
@@ -638,7 +719,8 @@ namespace BCG {
|
||||
myTableListBox.EndUpdate();
|
||||
}
|
||||
|
||||
private void testGridToolStripMenuItem_Click(object sender, EventArgs e) {
|
||||
private void testGridToolStripMenuItem_Click(object sender, EventArgs e)
|
||||
{
|
||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||
//while (true)
|
||||
//{
|
||||
@@ -660,9 +742,11 @@ namespace BCG {
|
||||
foreach (IBlueRow r in s.rows)
|
||||
rows.Add(r);
|
||||
|
||||
for (int i = 0; i < 100; i++) {
|
||||
for (int i = 0; i < 100; i++)
|
||||
{
|
||||
timer.start();
|
||||
rows.Sort(delegate (IBlueRow r1, IBlueRow r2) {
|
||||
rows.Sort(delegate (IBlueRow r1, IBlueRow r2)
|
||||
{
|
||||
return _sort_dicts_index(r1, r2, 5, true);
|
||||
});
|
||||
timer.stop();
|
||||
@@ -670,19 +754,22 @@ namespace BCG {
|
||||
}
|
||||
}
|
||||
|
||||
private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high) {
|
||||
private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high)
|
||||
{
|
||||
if (r1.Contains(column) && r2.Contains(column))
|
||||
return compare_send_dontcare_to_last(r1[column], r2[column], low_to_high);
|
||||
else
|
||||
return 0;
|
||||
}
|
||||
|
||||
private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high) {
|
||||
private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high)
|
||||
{
|
||||
//return compare_send_dontcare_to_last(r1[col_no], r2[col_no], low_to_high);
|
||||
return ((IComparable)r1[col_no]).CompareTo(r2[col_no]);
|
||||
}
|
||||
|
||||
internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) {
|
||||
internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high)
|
||||
{
|
||||
if (myIsDontCare(o1))
|
||||
return myIsDontCare(o2) ? 0 : 1;
|
||||
else
|
||||
@@ -690,14 +777,16 @@ namespace BCG {
|
||||
: ((IComparable)o2).CompareTo(o1));
|
||||
}
|
||||
|
||||
private static bool myIsDontCare(object o) {
|
||||
private static bool myIsDontCare(object o)
|
||||
{
|
||||
return o == null || o == DBNull.Value ||
|
||||
o is string && String.IsNullOrEmpty((string)o) ||
|
||||
o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) ||
|
||||
o is float && (Single.IsInfinity((float)o) || Single.IsNaN((float)o));
|
||||
}
|
||||
|
||||
private void _new_grid_form(object args) {
|
||||
private void _new_grid_form(object args)
|
||||
{
|
||||
//System.Timers.Timer t = new System.Timers.Timer();
|
||||
////t.SynchronizingObject
|
||||
//BCG.Utilities.BlueEventLog.logEvent(String.Format("New BTSTabularGridTestForm created on thread {0}", System.Threading.Thread.CurrentThread.ManagedThreadId),
|
||||
@@ -709,11 +798,13 @@ namespace BCG {
|
||||
|
||||
private void libraryToolStripMenuItem_Click(object sender, EventArgs e) { }
|
||||
|
||||
private void myTableFilterLabel_Click(object sender, EventArgs e) {
|
||||
private void myTableFilterLabel_Click(object sender, EventArgs e)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
private void MainForm_Load(object sender, EventArgs e) {
|
||||
private void MainForm_Load(object sender, EventArgs e)
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,16 +5,31 @@ using bts.utils;
|
||||
using mbts.config;
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Windows.Forms;
|
||||
using System.Threading.Tasks;
|
||||
using System.ComponentModel;
|
||||
|
||||
static class Program {
|
||||
[STAThread]
|
||||
static void Main(string[] args) {
|
||||
static void init(string[] args) {
|
||||
Func<string, string, Task> login = async (l, p) => {
|
||||
var login = l;
|
||||
var passwd = p;
|
||||
var service_ctx = bts.api.core.ServiceContext.create();
|
||||
await service_ctx.login(login, passwd);
|
||||
};
|
||||
BTSNativeMethods.provide_command_line(args, args.Length);
|
||||
Configuration.initialize();
|
||||
bts.Log.Gen.info($"Starting DSQA");
|
||||
Console.WriteLine("Starting DataStreamQueryAnalyzer...");
|
||||
Console.WriteLine("Starting Test Publisher");
|
||||
Task.Run(async () => await login("bts", String.Empty));
|
||||
}
|
||||
[STAThread]
|
||||
static void Main(string[] args) {
|
||||
init(args);
|
||||
string args_concat = String.Empty;
|
||||
args.ToList().ForEach(t => args_concat += t);
|
||||
var init_log = $"Starting DSQA args: {args_concat} len: {args.Length}";
|
||||
bts.Log.Gen.info(init_log);
|
||||
Console.WriteLine(init_log);
|
||||
Console.WriteLine("Launching Main Window");
|
||||
MainForm mainForm = new MainForm();
|
||||
if (args.Length > 0) {
|
||||
|
||||
+53
-97
@@ -9,8 +9,7 @@ using System.IO;
|
||||
using System.Threading;
|
||||
using mbts;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// Keeps an ITCPConnection for talking to the server. It also monitors heartbeats
|
||||
/// from the server, and in case of missing hbs, informs its clients, and tries to
|
||||
@@ -23,8 +22,7 @@ namespace BCG
|
||||
/// </summary>
|
||||
public class PubSubServerManagedTCPConnection : IPubSubServerConnection,
|
||||
ITCPMessageHandler,
|
||||
ITCPConnectionHandler
|
||||
{
|
||||
ITCPConnectionHandler {
|
||||
private ITCPConnection _connection, _new_connection;
|
||||
private readonly BlutoSubAddress _server_address;
|
||||
// if a message(with confirm) can not be sent , its id is queued here,
|
||||
@@ -43,98 +41,75 @@ namespace BCG
|
||||
public PubSubServerManagedTCPConnection(string serverURL)
|
||||
: this(new BlutoSubAddress(serverURL)) { }
|
||||
|
||||
public PubSubServerManagedTCPConnection(BlutoSubAddress server_address)
|
||||
{
|
||||
public PubSubServerManagedTCPConnection(BlutoSubAddress server_address) {
|
||||
_server_address = server_address;
|
||||
_connect_to_ps_server();
|
||||
}
|
||||
|
||||
private void _connect_to_ps_server()
|
||||
{
|
||||
private void _connect_to_ps_server() {
|
||||
bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server: {_server_address}");
|
||||
ClientToServerManagedTCPConnector.connect_to(_server_address, this, TimeSpan.FromSeconds(1));
|
||||
}
|
||||
|
||||
#region IPubSubServerConnection Members
|
||||
|
||||
public void sendWithConfirmation(byte[] data, ref string conf_id)
|
||||
{
|
||||
public void sendWithConfirmation(byte[] data, ref string conf_id) {
|
||||
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} myConnection : {_connection != null}");
|
||||
if (_connection != null)
|
||||
{
|
||||
_connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) =>
|
||||
{
|
||||
if (_connection != null) {
|
||||
_connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) => {
|
||||
if (_on_send_onfirm_callback != null)
|
||||
_on_send_onfirm_callback(msg_id, confirmed);
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
bts.Log.Gen.warn($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} no connection, buffering send");
|
||||
conf_id = Guid.NewGuid().ToString();
|
||||
_negative_unsent_confs.enqueue(conf_id);
|
||||
}
|
||||
}
|
||||
|
||||
public void send(byte[] data)
|
||||
{
|
||||
public void send(byte[] data) {
|
||||
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_address} data: {data.Length} myConnection : {_connection != null}");
|
||||
if (_connection != null)
|
||||
{
|
||||
if (_connection != null) {
|
||||
bts.Log.Gen.info("PubSubServerManagedTCPConnection.send server: " + _server_address + "data: " + data.Length);
|
||||
_connection.send(data);
|
||||
}
|
||||
}
|
||||
|
||||
public void send(Stream str)
|
||||
{
|
||||
public void send(Stream str) {
|
||||
BinaryReader br = new BinaryReader(str);
|
||||
send(br.ReadBytes(Convert.ToInt32(str.Length - str.Position)));
|
||||
}
|
||||
|
||||
public void send(PubSubMessage msg)
|
||||
{
|
||||
public void send(PubSubMessage msg) {
|
||||
send(msg.to_stream());
|
||||
}
|
||||
|
||||
public void send_with_confirmation(PubSubMessage msg, ref string conf_id)
|
||||
{
|
||||
public void send_with_confirmation(PubSubMessage msg, ref string conf_id) {
|
||||
var bytes = msg.to_bytes();
|
||||
sendWithConfirmation(bytes, ref conf_id);
|
||||
}
|
||||
|
||||
public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements)
|
||||
{
|
||||
public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements) {
|
||||
return _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements);
|
||||
}
|
||||
|
||||
public string serverURL
|
||||
{
|
||||
public string serverURL {
|
||||
get { return _server_address.ToString(); }
|
||||
}
|
||||
|
||||
public string clientURL
|
||||
{
|
||||
get
|
||||
{
|
||||
public string clientURL {
|
||||
get {
|
||||
string rv = "";
|
||||
lock (this)
|
||||
{
|
||||
if (_connection == null)
|
||||
{
|
||||
lock (this) {
|
||||
if (_connection == null) {
|
||||
rv = "";
|
||||
}
|
||||
else
|
||||
{
|
||||
if (PubSub3Config.instance.PS3_IPTranslation == null || PubSub3Config.instance.PS3_IPTranslation == String.Empty)
|
||||
{
|
||||
} else {
|
||||
if (PubSub3Config.instance.PS3_IPTranslation == null || PubSub3Config.instance.PS3_IPTranslation == String.Empty) {
|
||||
string ip;
|
||||
int port;
|
||||
_connection.getSelfAddress(out ip, out port);
|
||||
rv = new BlutoSubAddress(ip, port).ToString();
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
rv = "";
|
||||
}
|
||||
}
|
||||
@@ -143,18 +118,15 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public void registerCallback(ConnectionStatusChange del)
|
||||
{
|
||||
public void registerCallback(ConnectionStatusChange del) {
|
||||
_conn_callback = del;
|
||||
}
|
||||
|
||||
public void registerDataCallback(DataDelegate del)
|
||||
{
|
||||
public void registerDataCallback(DataDelegate del) {
|
||||
_data_callback = del;
|
||||
}
|
||||
|
||||
public void registerSendConfirmationCallback(SendConfirmationDelegate del)
|
||||
{
|
||||
public void registerSendConfirmationCallback(SendConfirmationDelegate del) {
|
||||
_on_send_onfirm_callback = del;
|
||||
}
|
||||
|
||||
@@ -163,21 +135,17 @@ namespace BCG
|
||||
#region IDisposable Members
|
||||
|
||||
private bool _is_disposed = false;
|
||||
public void Dispose()
|
||||
{
|
||||
public void Dispose() {
|
||||
bts.Log.Gen.info("PubSubServerManagedTCPConnection.Dispose");
|
||||
//throw new Exception("The method or operation is not implemented.");
|
||||
if (_connection != null)
|
||||
{
|
||||
if (_connection != null) {
|
||||
_connection.Dispose();
|
||||
|
||||
}
|
||||
_is_disposed = true;
|
||||
}
|
||||
public bool isDisposed
|
||||
{
|
||||
get
|
||||
{
|
||||
public bool isDisposed {
|
||||
get {
|
||||
return _is_disposed;
|
||||
}
|
||||
|
||||
@@ -193,12 +161,10 @@ namespace BCG
|
||||
/// (long) is safe for multi threaded operations
|
||||
/// </summary>
|
||||
/// <param name="message"></param>
|
||||
public void onData(RemoteMessage message)
|
||||
{
|
||||
public void onData(RemoteMessage message) {
|
||||
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData message: {message}");
|
||||
// TODO check for disconenction
|
||||
switch (message.messageType)
|
||||
{
|
||||
switch (message.messageType) {
|
||||
case RemoteMessageType.Data:
|
||||
if (_data_callback != null)
|
||||
_data_callback(message.remoteConnectionInfo.ToString(), message.message);
|
||||
@@ -219,17 +185,18 @@ namespace BCG
|
||||
break;
|
||||
|
||||
}
|
||||
if (message.messageType == RemoteMessageType.Data)
|
||||
{
|
||||
if (_data_callback != null)
|
||||
_data_callback(message.remoteConnectionInfo.ToString(), message.message);
|
||||
else
|
||||
_in_data_queue.enqueue(message.message);
|
||||
}
|
||||
else
|
||||
{
|
||||
_last_hb = DateTime.Now;
|
||||
}
|
||||
// this stupid shit, caused duplicating og messages and 2 days of wild goose chase
|
||||
// if (message.messageType == RemoteMessageType.Data)
|
||||
// {
|
||||
// if (_data_callback != null)
|
||||
// _data_callback(message.remoteConnectionInfo.ToString(), message.message);
|
||||
// else
|
||||
// _in_data_queue.enqueue(message.message);
|
||||
// }
|
||||
// else
|
||||
// {
|
||||
// _last_hb = DateTime.Now;
|
||||
// }
|
||||
}
|
||||
|
||||
#endregion
|
||||
@@ -241,15 +208,13 @@ namespace BCG
|
||||
/// except onData)
|
||||
/// </summary>
|
||||
/// <param name="connection"></param>
|
||||
public void onConnection(ITCPConnection connection)
|
||||
{
|
||||
public void onConnection(ITCPConnection connection) {
|
||||
string ip;
|
||||
int port;
|
||||
connection.getPeerAddress(out ip, out port);
|
||||
BlutoSubAddress add = new BlutoSubAddress(ip, port);
|
||||
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onConnection add: {add.ToString()} connection type: {connection.GetType()}");
|
||||
if (add.ToString() == _server_address.ToString())
|
||||
{
|
||||
if (add.ToString() == _server_address.ToString()) {
|
||||
_last_hb = DateTime.Now;
|
||||
_new_connection = connection;
|
||||
}
|
||||
@@ -257,39 +222,31 @@ namespace BCG
|
||||
|
||||
#endregion
|
||||
|
||||
void _dispose_and_reconnect_if_missing_hb()
|
||||
{
|
||||
if (!_connection.isDisposed)
|
||||
{
|
||||
void _dispose_and_reconnect_if_missing_hb() {
|
||||
if (!_connection.isDisposed) {
|
||||
TimeSpan hb_interval = DateTime.Now - _last_hb;
|
||||
bool hb_missed = hb_interval > _connection_drop_threshold;
|
||||
if (hb_missed)
|
||||
{
|
||||
if (hb_missed) {
|
||||
if (_conn_callback != null)
|
||||
_conn_callback(_server_address.ToString(), false);
|
||||
bts.Log.Gen.info("PubSubServerManagedTCPConnection._dispose_and_reconnect_if_missing_hb disposing connecion: {myServerAddress}");
|
||||
_connection.Dispose();
|
||||
_connect_to_ps_server();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
// already disposed, nothing to do
|
||||
}
|
||||
}
|
||||
|
||||
void _check_for_new_connection()
|
||||
{
|
||||
if (_new_connection != null)
|
||||
{
|
||||
void _check_for_new_connection() {
|
||||
if (_new_connection != null) {
|
||||
_connection = _new_connection;
|
||||
_new_connection = null; // since we are using its presence as a marker of nw connections
|
||||
bts.Log.Gen.info("PubSubServerManagedTCPConnection._check_for_new_connection assigning new connection");
|
||||
_connection.register(this);
|
||||
if (_conn_callback != null)
|
||||
_conn_callback(_server_address.ToString(), true);
|
||||
if (_on_send_onfirm_callback != null)
|
||||
{
|
||||
if (_on_send_onfirm_callback != null) {
|
||||
var msg_ids = _negative_unsent_confs.dequeueAll();
|
||||
foreach (var id in msg_ids)
|
||||
_on_send_onfirm_callback(id, false);
|
||||
@@ -304,8 +261,7 @@ namespace BCG
|
||||
/// the design only allows for onConnection to be on a different
|
||||
/// thread.
|
||||
/// </summary>
|
||||
public void doWork()
|
||||
{
|
||||
public void doWork() {
|
||||
//_dispose_and_reconnect_if_missing_hb();
|
||||
_check_for_new_connection();
|
||||
///check for heartbeats and let clients know in
|
||||
|
||||
@@ -304,6 +304,7 @@ namespace BCG {
|
||||
}
|
||||
|
||||
private void myCollectMissingHeartbeatConnections() {
|
||||
return; // disabling HBs for now to simplify things for the demo
|
||||
if (DateTime.Now - myLastHeartbeatDeadCollectionTime > mySubscriberDropThreshold) {
|
||||
myLastHeartbeatDeadCollectionTime = DateTime.Now;
|
||||
///these are all the hbs received bw the last
|
||||
@@ -321,10 +322,9 @@ namespace BCG {
|
||||
if (!indexedHbs.ContainsKey(clientURL))
|
||||
deadClients.Add(clientURL);
|
||||
foreach (string deadClient in deadClients) {
|
||||
string disposeMsg = String.Format("Collecting missing HB connections : {0} is dead due to missing hbs, Disposing it now, " +
|
||||
string dispose_msg = String.Format("Collecting missing HB connections : {0} is dead due to missing hbs, Disposing it now, " +
|
||||
"connection count before deleting this = {1}", deadClient, myConnections.Count);
|
||||
//BlueEventLog.logEvent(disposeMsg, PS3LogLevels.disconnection);
|
||||
Console.WriteLine(disposeMsg);
|
||||
bts.Log.Gen.error(dispose_msg);
|
||||
myConnections[deadClient].Dispose();
|
||||
myConnections.Remove(deadClient);
|
||||
myDeadConnectionURLs.enqueue(deadClient);
|
||||
|
||||
@@ -181,7 +181,7 @@ namespace BCG {
|
||||
|
||||
public void publish(string tagType, string tag, Stream content) {
|
||||
PublisherCoreMessage msg = new PublisherCoreMessage(tagType, tag, content);
|
||||
bts.Log.Gen.info($"StaticPublisherCore.publish msg: {msg}");
|
||||
bts.Log.Gen.info($"StaticPublisherCore.publish tag: {tag} msg: {msg} len: {content.Length - content.Position}");
|
||||
myMainQueue.enqueue(msg);
|
||||
}
|
||||
|
||||
@@ -357,7 +357,7 @@ namespace BCG {
|
||||
myState.publish(tag, tagType, content);
|
||||
if (!myCrushableTagsWaitingOnAck.ContainsKey(tag)) {
|
||||
bts.Log.Gen.info("Sending crushable tag: {tag}");
|
||||
mySendData(tag, tagType, content);
|
||||
mySendData(tag, tagType, content, "myPublish");
|
||||
} else {
|
||||
bts.Log.Gen.info("Could not send tag: {tag} since its waiting on an ack");
|
||||
myCrushableTagsWaitingOnAck[tag].second++;
|
||||
@@ -374,8 +374,8 @@ namespace BCG {
|
||||
}
|
||||
|
||||
// add header, possibly compress the data, and sendto appropriate pubsub
|
||||
private void mySendData(string tag, string tagType, Stream content) {
|
||||
bts.Log.Gen.info($"StaticPublisherCore.mySendData tag: {tag} tagType: {tagType}");
|
||||
private void mySendData(string tag, string tagType, Stream content, string reason) {
|
||||
bts.Log.Gen.info($"StaticPublisherCore.mySendData tag: {tag} tagType: {tagType} reason: {reason} len: {content.Length - content.Position}");
|
||||
long initPos = content.Position;
|
||||
// locate right pubsub for this tag
|
||||
IPubSubServerConnection serverConn = myGetOrBuildSender(tag);
|
||||
@@ -407,7 +407,6 @@ namespace BCG {
|
||||
serverConn.send_with_confirmation(ps_msg, ref refId);
|
||||
myCrushableTagsWaitingOnAck.Add(tag, new Pair<string, int>(refId, 0));
|
||||
} else {
|
||||
//Console.WriteLine("StaticPublisherCore.mySend serverConn.send tag: " + tag + " server: " + serverConn.serverURL);
|
||||
// this is where the data is actually sent out
|
||||
var ps_msg = new PubSubMessage(pub_md, contentToPublish);
|
||||
bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
|
||||
@@ -440,6 +439,7 @@ namespace BCG {
|
||||
///in case of the server coming up, it publishes all the stored state
|
||||
///for that server, back to it
|
||||
protected void myHandleServerReboot(string serverURL, bool connection) {
|
||||
bts.Log.Gen.info($"StaticPublisherCore.myHandleServerReboot server_url: {serverURL}");
|
||||
if (connection) {
|
||||
string thisPublisherInfo = ClientInfo.thisClient.processString;
|
||||
foreach (KeyValuePair<string, IPubSubServerConnection> p in myTagToSender) {
|
||||
@@ -448,7 +448,7 @@ namespace BCG {
|
||||
string tag = p.Key;
|
||||
myCrushableTagsWaitingOnAck.Clear();
|
||||
byte[] data = myState.getContent(tag, out tagType);
|
||||
mySendData(tag, tagType, new MemoryStream(data));
|
||||
mySendData(tag, tagType, new MemoryStream(data), "myHandleServerReboot");
|
||||
|
||||
//Stream content = new MemoryStream(data);
|
||||
//Stream contentToPublish = content;
|
||||
|
||||
@@ -13,8 +13,7 @@ using mbts;
|
||||
using mbts.spark;
|
||||
//using BCG.Utilities.BlueConfiguration;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
namespace BCG {
|
||||
|
||||
/// <summary>
|
||||
/// This class serves as the subscrition interface to the pubsub. The only methods
|
||||
@@ -25,70 +24,56 @@ namespace BCG
|
||||
/// to the servers even though there might be so created by other instances of the SubscriberCore
|
||||
///
|
||||
/// </summary>
|
||||
public class StaticSubscriberCore : ISubscriberCore
|
||||
{
|
||||
public class StaticSubscriberCore : ISubscriberCore {
|
||||
private Dictionary<string, ISingleServerSubscriberCore> myServerURLToSubCore = new Dictionary<string, ISingleServerSubscriberCore>();
|
||||
private IPubSubRouter myRouter;
|
||||
private HighPerformanceTimer myTimer = new HighPerformanceTimer();
|
||||
private PubSubServerConnectionFactory myConnectionFactory = new PubSubServerConnectionFactory(true);
|
||||
|
||||
public event PubSubConnectionChange onPubSubConnectionChange;
|
||||
|
||||
|
||||
|
||||
|
||||
#region constructors
|
||||
|
||||
public StaticSubscriberCore()
|
||||
{
|
||||
public StaticSubscriberCore() {
|
||||
//this defaults to a basic pubsubrouter if the config key is not filled out
|
||||
myInit(myGetRouter());
|
||||
}
|
||||
public StaticSubscriberCore(string siteName)
|
||||
{
|
||||
public StaticSubscriberCore(string siteName) {
|
||||
myInit(new PubSubRouter(PubSub3Config.instance.getRouting(siteName)));
|
||||
}
|
||||
|
||||
private IPubSubRouter myGetRouter()
|
||||
{return new PubSubRouter(PubSub3Config.instance.getRouting());}
|
||||
private IPubSubRouter myGetRouter() { return new PubSubRouter(PubSub3Config.instance.getRouting()); }
|
||||
|
||||
public StaticSubscriberCore(PubSubRoute[] routes)
|
||||
{
|
||||
public StaticSubscriberCore(PubSubRoute[] routes) {
|
||||
myInit(new PubSubRouter(routes));
|
||||
}
|
||||
|
||||
public StaticSubscriberCore(IPubSubRouter router)
|
||||
{
|
||||
public StaticSubscriberCore(IPubSubRouter router) {
|
||||
myInit(router);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
private void myInit(IPubSubRouter router)
|
||||
{
|
||||
private void myInit(IPubSubRouter router) {
|
||||
myRouter = router;
|
||||
}
|
||||
|
||||
public ArrayList getUpdates()
|
||||
{
|
||||
public ArrayList getUpdates() {
|
||||
return getUpdates(10);
|
||||
}
|
||||
|
||||
public ArrayList getUpdates(TimeSpan timeout)
|
||||
{
|
||||
public ArrayList getUpdates(TimeSpan timeout) {
|
||||
return getUpdates(Convert.ToInt32(timeout.TotalMilliseconds));
|
||||
}
|
||||
|
||||
public ArrayList getUpdates(int timeout)
|
||||
{
|
||||
public ArrayList getUpdates(int timeout) {
|
||||
//BCG.BlueTrace.WriteTraceMethodEnterW("StaticSubscriberCore.getUpdates");
|
||||
lock (myServerURLToSubCore)
|
||||
{
|
||||
lock (myServerURLToSubCore) {
|
||||
|
||||
if (myServerURLToSubCore.Count == 0)
|
||||
{
|
||||
if (myServerURLToSubCore.Count == 0) {
|
||||
return new ArrayList();
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
///gets a time share for every subscriber and
|
||||
///calls their updates
|
||||
myTimer.start();
|
||||
@@ -96,6 +81,7 @@ namespace BCG
|
||||
List<PubSubUpdate> retVal = new List<PubSubUpdate>();
|
||||
foreach (ISingleServerSubscriberCore subCore in myServerURLToSubCore.Values)
|
||||
retVal.AddRange(subCore.getUpdates(timeShare));
|
||||
_log_n_updates(retVal);
|
||||
myTimer.stop();
|
||||
//if (myTimer.lastIntervalTime > 10)
|
||||
// BlueEventLog.logEvent("get updates taking " + myTimer.lastIntervalTime, BlueEventLevel.Error);
|
||||
@@ -104,30 +90,36 @@ namespace BCG
|
||||
return new ArrayList(retVal);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
private Dictionary<string,bool> myBlueCommTags = new Dictionary<string,bool> ();
|
||||
|
||||
bool myIsBlueCommTag(string tag)
|
||||
{
|
||||
return false;
|
||||
private IDictionary<string, int> _n_update_map = new Dictionary<string, int>();
|
||||
private void _log_n_updates(IEnumerable<PubSubUpdate> updates) {
|
||||
foreach (PubSubUpdate u in updates) {
|
||||
if (!_n_update_map.ContainsKey(u.tag)) {
|
||||
_n_update_map.Add(u.tag, 0);
|
||||
}
|
||||
_n_update_map[u.tag] = _n_update_map[u.tag] + 1;
|
||||
}
|
||||
// log it now
|
||||
foreach (var kv in _n_update_map) {
|
||||
bts.Log.Gen.info($"StaticSubscrberCore tag: {kv.Key} n_updates: {kv.Value}");
|
||||
}
|
||||
}
|
||||
|
||||
public void subscribe(params string[] tags)
|
||||
{
|
||||
lock (myServerURLToSubCore)
|
||||
{
|
||||
foreach (string tag in tags)
|
||||
{
|
||||
if (myIsBlueCommTag(tag))
|
||||
{
|
||||
}
|
||||
else
|
||||
{
|
||||
private Dictionary<string, bool> myBlueCommTags = new Dictionary<string, bool>();
|
||||
|
||||
bool myIsBlueCommTag(string tag) {
|
||||
return false;
|
||||
}
|
||||
|
||||
public void subscribe(params string[] tags) {
|
||||
lock (myServerURLToSubCore) {
|
||||
foreach (string tag in tags) {
|
||||
if (myIsBlueCommTag(tag)) {
|
||||
} else {
|
||||
string serverURL = myRouter.getSubURLForTag(tag);
|
||||
if (!myServerURLToSubCore.ContainsKey(serverURL))
|
||||
{
|
||||
if (!myServerURLToSubCore.ContainsKey(serverURL)) {
|
||||
// BlueEventLog.logEvent("Creating a SingleServerSubscriberCore for " + serverURL, BlueEventLevel.Information);
|
||||
myServerURLToSubCore.Add(serverURL, new SingleServerSubscriberCore(serverURL, myConnectionFactory));
|
||||
myServerURLToSubCore[serverURL].onPubSubConnectionChange += myHandleConnectionChange;
|
||||
@@ -138,28 +130,22 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
}
|
||||
public void myHandleConnectionChange (bool isConnected, string connection)
|
||||
{
|
||||
public void myHandleConnectionChange(bool isConnected, string connection) {
|
||||
if (onPubSubConnectionChange != null)
|
||||
onPubSubConnectionChange(isConnected, connection);
|
||||
|
||||
}
|
||||
|
||||
public void unsubscribe(params string[] tags)
|
||||
{
|
||||
lock (myServerURLToSubCore)
|
||||
{
|
||||
public void unsubscribe(params string[] tags) {
|
||||
lock (myServerURLToSubCore) {
|
||||
|
||||
foreach (string tag in tags)
|
||||
{
|
||||
foreach (string tag in tags) {
|
||||
if (myIsBlueCommTag(tag))
|
||||
myServerURLToSubCore["bluecomm"].unsubscribe(tag);
|
||||
else
|
||||
{
|
||||
else {
|
||||
string serverURL = myRouter.getSubURLForTag(tag);
|
||||
|
||||
if (!myServerURLToSubCore.ContainsKey(serverURL))
|
||||
{
|
||||
if (!myServerURLToSubCore.ContainsKey(serverURL)) {
|
||||
// BlueEventLog.logEvent("Creating a SingleServerSubscriberCore for " + serverURL, BlueEventLevel.Information);
|
||||
myServerURLToSubCore.Add(serverURL, new SingleServerSubscriberCore(serverURL, myConnectionFactory));
|
||||
}
|
||||
@@ -168,15 +154,13 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
}
|
||||
public IPubSubRouter getRouter()
|
||||
{
|
||||
public IPubSubRouter getRouter() {
|
||||
return myRouter;
|
||||
}
|
||||
|
||||
#region IDisposable Members
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
public void Dispose() {
|
||||
|
||||
foreach (ISingleServerSubscriberCore sssc in myServerURLToSubCore.Values)
|
||||
sssc.Dispose();
|
||||
|
||||
@@ -40,7 +40,7 @@ namespace BCG {
|
||||
IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
|
||||
MarketDataClient _md_client;
|
||||
HighPerformanceTimer _timer = new HighPerformanceTimer();
|
||||
private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
|
||||
private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("books",
|
||||
Tuple.Create("id", TypeCode.Int32),
|
||||
Tuple.Create("sec_key", TypeCode.UInt32),
|
||||
Tuple.Create("level", TypeCode.UInt32),
|
||||
|
||||
+76
-31
@@ -1,20 +1,26 @@
|
||||
namespace BCG {
|
||||
namespace BCG
|
||||
{
|
||||
using bts.api.orderman;
|
||||
|
||||
enum TSGroupByOperation {
|
||||
enum TSGroupByOperation
|
||||
{
|
||||
OHLC = 0,
|
||||
Statistics
|
||||
}
|
||||
|
||||
[DataStreamUserRespresentation("timeseriesanalysis")]
|
||||
public class TimeSeriesGroupByStream : ViewStream {
|
||||
public TimeSeriesGroupByStream(ViewStream md_str, string timestamp_col, string output_col, string operation) {
|
||||
ViewStream rs = new NonIncGroupByStream(md_str, new OHLCNonIncGroupByEvaluator(timestamp_col, output_col));
|
||||
public class TimeSeriesGroupByStream : ViewStream
|
||||
{
|
||||
public TimeSeriesGroupByStream(ViewStream md_str, string timebucket_col, string output_col, string operation)
|
||||
{
|
||||
ViewStream rs = new NonIncGroupByStream(md_str, new OHLCNonIncGroupByEvaluator(timebucket_col, output_col));
|
||||
mySetRealSource(rs);
|
||||
}
|
||||
}
|
||||
public class OHLC {
|
||||
public OHLC(double open_args, double high_args, double low_args, double close_args) {
|
||||
public class OHLC
|
||||
{
|
||||
public OHLC(double open_args, double high_args, double low_args, double close_args)
|
||||
{
|
||||
open = open_args;
|
||||
high = high_args;
|
||||
low = low_args;
|
||||
@@ -22,36 +28,47 @@ namespace BCG {
|
||||
}
|
||||
public readonly double open, high, low, close;
|
||||
}
|
||||
public class OHLCNonIncGroupByEvaluator : INonIncGroupByEvaluator {
|
||||
public class OHLCNonIncGroupByEvaluator : INonIncGroupByEvaluator
|
||||
{
|
||||
private string _timestamp_col;
|
||||
private string _output_col;
|
||||
private IBlueTableSchema _out_schema;
|
||||
public OHLCNonIncGroupByEvaluator(string timestamp_col, string output_col) {
|
||||
public OHLCNonIncGroupByEvaluator(string timestamp_col, string output_col)
|
||||
{
|
||||
_timestamp_col = timestamp_col;
|
||||
_output_col = output_col;
|
||||
}
|
||||
|
||||
string open_col {
|
||||
get {
|
||||
string open_col
|
||||
{
|
||||
get
|
||||
{
|
||||
return $"{_output_col}_open"; ;
|
||||
}
|
||||
}
|
||||
string high_col {
|
||||
get {
|
||||
string high_col
|
||||
{
|
||||
get
|
||||
{
|
||||
return $"{_output_col}_high"; ;
|
||||
}
|
||||
}
|
||||
string low_col {
|
||||
get {
|
||||
string low_col
|
||||
{
|
||||
get
|
||||
{
|
||||
return $"{_output_col}_low"; ;
|
||||
}
|
||||
}
|
||||
string close_col {
|
||||
get {
|
||||
string close_col
|
||||
{
|
||||
get
|
||||
{
|
||||
return $"{_output_col}_close"; ;
|
||||
}
|
||||
}
|
||||
private void _build_schema(IBlueTableSchema input_schema) {
|
||||
private void _build_schema(IBlueTableSchema input_schema)
|
||||
{
|
||||
List<Tuple<Type, string>> new_cols = new List<Tuple<Type, string>>();
|
||||
ColumnInfo ci = input_schema[_output_col];
|
||||
new_cols.Add(Tuple.Create(ci.colType, open_col));
|
||||
@@ -62,23 +79,30 @@ namespace BCG {
|
||||
_out_schema = BlueDataUtility.add_columns(input_schema, new_cols.ToArray());
|
||||
}
|
||||
|
||||
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
|
||||
if (_out_schema == null) {
|
||||
if (grouped_rows.Count > 0) {
|
||||
foreach (var kv in grouped_rows) {
|
||||
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows)
|
||||
{
|
||||
if (_out_schema == null)
|
||||
{
|
||||
if (grouped_rows.Count > 0)
|
||||
{
|
||||
foreach (var kv in grouped_rows)
|
||||
{
|
||||
var rows = kv.Value;
|
||||
if (rows != null && rows.Count > 0) {
|
||||
if (rows != null && rows.Count > 0)
|
||||
{
|
||||
_build_schema(rows.ElementAt(0).schema);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
BlueTable rv = new BlueTable(_out_schema);
|
||||
foreach (var kv in grouped_rows) {
|
||||
foreach (var kv in grouped_rows)
|
||||
{
|
||||
int id = kv.Key;
|
||||
IEnumerable<IBlueRow> rows = kv.Value;
|
||||
// sort rows on time to compute o,c correctly??
|
||||
if (rows != null && rows.Count() > 0) {
|
||||
if (rows != null && rows.Count() > 0)
|
||||
{
|
||||
IBlueRow reprow = rows.ElementAt(0);
|
||||
IEnumerable<double> vals = BlueDataUtility.get_covalues<double>(rows, _output_col);
|
||||
OHLC ohlc = compute_ohlc(vals);
|
||||
@@ -94,11 +118,13 @@ namespace BCG {
|
||||
return rv;
|
||||
}
|
||||
|
||||
public string[] get_group_by_cols() {
|
||||
public string[] get_group_by_cols()
|
||||
{
|
||||
return new string[] { _timestamp_col };
|
||||
}
|
||||
|
||||
static OHLC compute_ohlc(IEnumerable<double> prices) {
|
||||
static OHLC compute_ohlc(IEnumerable<double> prices)
|
||||
{
|
||||
double o = prices.ElementAt(0);
|
||||
double c = prices.ElementAt(prices.Count() - 1);
|
||||
double l = prices.Min();
|
||||
@@ -106,11 +132,30 @@ namespace BCG {
|
||||
return new(o, h, l, c);
|
||||
}
|
||||
}
|
||||
[DataStreamUserRespresentation("ohlc")]
|
||||
public class OHLCStream : ViewStream {
|
||||
[DataStreamUserRespresentation("RSI")]
|
||||
public class RSIStream : ViewStream
|
||||
{
|
||||
// for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
|
||||
public OHLCStream(ViewStream md_str, string timestamp_col, string output_col) {
|
||||
ViewStream rs = new TimeSeriesGroupByStream(md_str, timestamp_col, output_col, TSGroupByOperation.OHLC.ToString());
|
||||
public RSIStream(ViewStream md_str, string timebucket_column, string output_col)
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
[DataStreamUserRespresentation("Bollinger")]
|
||||
public class BollingerBandStream : ViewStream
|
||||
{
|
||||
// for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
|
||||
public BollingerBandStream(ViewStream md_str, string timebucket_column, string output_col)
|
||||
{
|
||||
}
|
||||
}
|
||||
[DataStreamUserRespresentation("OHLC")]
|
||||
public class OHLCStream : ViewStream
|
||||
{
|
||||
// for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
|
||||
public OHLCStream(ViewStream md_str, string timebucket_column, string output_col)
|
||||
{
|
||||
ViewStream rs = new TimeSeriesGroupByStream(md_str, timebucket_column, output_col, TSGroupByOperation.OHLC.ToString());
|
||||
mySetRealSource(rs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
namespace BCG {
|
||||
namespace BCG
|
||||
{
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using bts.api;
|
||||
@@ -19,7 +20,8 @@ namespace BCG {
|
||||
using SecurityKey = System.UInt32;
|
||||
|
||||
[DataStreamUserRespresentation("timesales")]
|
||||
public class TimeSalesStream : ViewStream, IWorker {
|
||||
public class TimeSalesStream : ViewStream, IWorker
|
||||
{
|
||||
private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
|
||||
private DateTime _last_poll_time = DateTime.MinValue;
|
||||
private ViewStreamDatabase _vsd;
|
||||
@@ -28,37 +30,43 @@ namespace BCG {
|
||||
MarketDataClient _md_client;
|
||||
private bool _dirty = false;
|
||||
private ConcurrentQueue<TradeInfo> _cached_trades = new ConcurrentQueue<TradeInfo>();
|
||||
private BlueTable _table = new BlueTable(BlueDataUtility.form_schema("instruments",
|
||||
private readonly int _buffer_size;
|
||||
private IBlueTableSchema _schema = BlueDataUtility.form_schema("TimeSales",
|
||||
Tuple.Create("id", TypeCode.Int32),
|
||||
Tuple.Create("sec_key", TypeCode.UInt32),
|
||||
Tuple.Create("sec", TypeCode.String),
|
||||
Tuple.Create("time", TypeCode.Int64),
|
||||
Tuple.Create("time", TypeCode.DateTime),
|
||||
Tuple.Create("price", TypeCode.Double),
|
||||
Tuple.Create("qty", TypeCode.UInt32),
|
||||
Tuple.Create("aggressor", TypeCode.String)
|
||||
));
|
||||
public TimeSalesStream()
|
||||
: this(ViewStreamDatabase.instance) {
|
||||
Tuple.Create("aggressor", TypeCode.String));
|
||||
public TimeSalesStream(int buffer_size)
|
||||
: this(buffer_size, ViewStreamDatabase.instance)
|
||||
{
|
||||
}
|
||||
|
||||
public TimeSalesStream(ViewStreamDatabase vsd) {
|
||||
public TimeSalesStream(int buffer_size, ViewStreamDatabase vsd)
|
||||
{
|
||||
_buffer_size = buffer_size;
|
||||
bts.Log.Gen.info("TimeSalesStream.cons");
|
||||
_vsd = vsd;
|
||||
_md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
_vsd.addWorker(this);
|
||||
_md_client.trades(new TradesSubscriptionFilter()).Subscribe(_on_trades);
|
||||
//_md_client.trades(null).Subscribe(_on_trades);
|
||||
|
||||
}
|
||||
|
||||
private void _log_updates(IList<TradeInfo> trades) {
|
||||
private void _log_updates(IList<TradeInfo> trades)
|
||||
{
|
||||
bts.Log.Gen.info($"TimeSalesStream update sz: {trades.Count}");
|
||||
}
|
||||
|
||||
private void _on_trades(IList<TradeInfo> trades) {
|
||||
private void _on_trades(IList<TradeInfo> trades)
|
||||
{
|
||||
_dirty = true;
|
||||
_log_updates(trades);
|
||||
// just cacching in here, will populate the rows in _poll_scan and fire the datapacket
|
||||
@@ -67,23 +75,35 @@ namespace BCG {
|
||||
_cached_trades.Enqueue(t);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
_vsd.removeWorker(this);
|
||||
}
|
||||
|
||||
private void _scan_trades() {
|
||||
BlueTable added_rows = new BlueTable(_table.schema);
|
||||
foreach (TradeInfo tinf in _cached_trades) {
|
||||
IBlueRow r = new BlueRow(_table.schema, _id++, tinf.security.Id, tinf.security.ToString(), tinf.trade.Time, tinf.trade.Mantissa, tinf.trade.Size, tinf.trade.AggressorSide.ToString());
|
||||
added_rows.addRow(r);
|
||||
private void _scan_trades()
|
||||
{
|
||||
Func<TradeInfo, DateTime> to_dt = (tinf) =>
|
||||
{
|
||||
return new mbts.utils.LLDateTime((ulong)tinf.trade.Time).to_datetime().ToLocalTime();
|
||||
};
|
||||
|
||||
List<TradeInfo> trades = new List<TradeInfo>(_cached_trades);
|
||||
var latest_n_trades = trades.OrderByDescending(t => (ulong)t.trade.Time).Take(_buffer_size).ToList();
|
||||
BlueTable new_state = new BlueTable(_schema);
|
||||
foreach (TradeInfo tinf in Enumerable.Reverse(latest_n_trades))
|
||||
{
|
||||
// TODO, time
|
||||
IBlueRow r = new BlueRow(_schema, _id++, tinf.security.Id, tinf.security.ToString(), to_dt(tinf), tinf.trade.Mantissa * Math.Pow(10, tinf.trade.Exponent), tinf.trade.Size, tinf.trade.AggressorSide.ToString());
|
||||
new_state.addRow(r);
|
||||
}
|
||||
_table.addRows(added_rows);
|
||||
if (added_rows.Count > 0)
|
||||
myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
|
||||
DataPacket dp = BlueDataUtility.build_dp_inc(new_state, myLastValue as DataPacket);
|
||||
myFireEvent(dp);
|
||||
}
|
||||
|
||||
public void doWork() {
|
||||
if (_dirty && DateTime.Now - _last_poll_time > _poll_period) {
|
||||
public void doWork()
|
||||
{
|
||||
if (_dirty && DateTime.Now - _last_poll_time > _poll_period)
|
||||
{
|
||||
_last_poll_time = DateTime.Now;
|
||||
_dirty = false;
|
||||
_scan_trades();
|
||||
|
||||
@@ -45,7 +45,7 @@ namespace BCG
|
||||
|
||||
#region constructors to be used by the static factory method
|
||||
|
||||
private PubSubServerTagStatsStream(string serverURL)
|
||||
public PubSubServerTagStatsStream(string serverURL)
|
||||
: this(ViewStreamDatabase.instance, serverURL)
|
||||
{ }
|
||||
|
||||
|
||||
Reference in new issue
Block a user