TimeSeries and TimeBucketStreams, Stream offloading etc

This commit is contained in:
Taha committed 2023-10-23 14:18:31 -04:00
1 parent 33030b1212
commit 88bf04e5f1
31 files changed
+1823 -836

No files matched your search

@@ -0,0 +1,24 @@
using BCG;
namespace BCG
{
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
[DataStreamUserRespresentation("add_to_cloud")]
public class AddToCloudStream : ViewStream
{
private readonly string _name, _stream_exp;
private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
public AddToCloudStream(string name, string stream_exp)
: this(name, stream_exp, 0)
{
}
public AddToCloudStream(string name, string stream_exp, int persist)
{
_name = name;
_stream_exp = stream_exp;
_requestor.add_to_cloud(_name, _stream_exp, persist == 1);
mySetRealSource(new TableStream("cloud_streams"));
}
}
}
@@ -0,0 +1,100 @@
using BCG;
using System;
using System.Collections.Generic;
using mbts.config;
using bts.utils;
using System.Threading.Tasks;
using Bts.DataStreamExpressions;
using System.Collections.Generic;
using BCG.Utilities.General;
namespace BCG
{
// listens to the requests posted on "cloud_streams" request table, and
// if unfulfilled starts a ViewPubsliher
public class CloudStreamManager : IWorker
{
public class StreamPubUnit
{
public string name;
public string stream_expression;
public ViewStream stream;
public bool persist;
public ViewPublisher view_publisher;
public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
{
name = name_;
stream_expression = stream_expression_;
stream = stream_;
persist = persist_;
view_publisher = view_publisher_;
}
}
static ViewStream build_stream(string stream_expression)
{
try
{
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv;
}
catch (Exception ex)
{
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
return null;
}
}
private IDictionary<string, StreamPubUnit>
_stream_pub_map = new Dictionary<string, StreamPubUnit>();
private ViewStream _stream = new TableStream("cloud_streams");
private PubTableBroker _broker = new PubTableBroker();
public CloudStreamManager()
{
_stream.onChange += _on_str;
}
private void _on_str(object obj)
{
DataPacket dp = obj as DataPacket;
foreach (IBlueRow r in dp.newRows)
{
string name = r.get<string>("name");
bool persist = r.get<bool>("persist");
string stream_exp = r.get<string>("stream_exp");
if (!_stream_pub_map.ContainsKey(name))
{
_assign(name, stream_exp, persist);
}
}
}
private void _assign(string name, string stream_expression, bool persist)
{
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
ViewStream stream = build_stream(stream_expression);
_expose_to_ws(name, stream);
if (persist)
{
stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs
}
ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
_stream_pub_map.Add(name, u);
}
private void _expose_to_ws(string name, ViewStream stream)
{
// TODO: john
}
public void doWork()
{
}
}
}
@@ -0,0 +1,59 @@
using BCG;
using System.Linq;
namespace BCG
{
public class StreamOffloadRequestor
{
public class OffloadOp
{
string name;
string stream_exp;
}
private IBlueTableSchema _schema;
private IBlueRowCollection _rows;
private PubTableBroker _broker = new PubTableBroker();
ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
public StreamOffloadRequestor()
{
_schema = new BlueTableSchema("cloud_streams",
new ColumnInfo("id", 0, typeof(int)),
new ColumnInfo("name", 1, typeof(string)),
new ColumnInfo("persist", 2, typeof(bool)),
new ColumnInfo("stream_exp", 3, typeof(string)));
_existing_offloads_vs.onChange += _on_str;
}
private void _on_str(object obj)
{
DataPacket dp = obj as DataPacket;
_rows = dp.newRows;
// this is needed so the broker knows the state of the table
_broker.createTable(_rows);
}
public void add_to_cloud(string name, string stream_exp, bool persist)
{
// check if it exists
if (_rows == null)
{
BlueTable table = new BlueTable(_schema);
IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
table.addRow(nr);
_broker.createTable(table);
// create table
}
else
{
if (exists(name))
return;
IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
_broker.addRow(nr);
}
}
bool exists(string name)
{
return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
}
}
}
@@ -8,6 +8,7 @@ using mbts.spark;
namespace BCG
{
[DataStreamUserRespresentation("fastJoin")]
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
{
private readonly NonIncJoinSpec[] _join_specs;
@@ -23,7 +24,7 @@ namespace BCG
public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
ViewStream right, string right_join_cols, string right_proj)
: this(new Triplet<ViewStream, string, string>[]
: this(new Triplet<ViewStream, string, string>[]
{
new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
@@ -31,7 +32,7 @@ namespace BCG
{ }
public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
: this (NonIncJoinSpec.from_triplet (join_specs), null, 0)
: this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
{ }
public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
@@ -102,8 +103,8 @@ namespace BCG
{
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
d.constribute_cols_to_schema (cis);
_output_schema = new BlueTableSchema ("NonIncJoinStream", cis.ToArray());
d.constribute_cols_to_schema(cis);
_output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
}
}
@@ -24,6 +24,8 @@ namespace BCG {
private void _onstream_change(object o) {
DataPacket dp = o as DataPacket;
BlueDataUtility.validate_table_schema(dp.newRows);
BlueDataUtility.validate_table_schema(dp.addedRows);
if (_last_recv_rows == null) {
_last_recv_rows = dp.newRows;
} else {
@@ -1,37 +1,47 @@
using Google.Protobuf.WellKnownTypes;
using System;
using BCG.Utilities.General;
namespace BCG {
namespace BCG
{
// Keeps adding changing rows to the stream, hence preserving the history
// and creating a time series, for example this can be used to make bbo updates
// into a time series of prices/qty
public class TimeSeriesStream : ViewStream {
public class TimeSeriesStream : ViewStream
{
private BlueTable _table;
private ViewStream _stream;
private int _id = 0;
public TimeSeriesStream(ViewStream stream) {
public TimeSeriesStream(ViewStream stream)
{
_stream = stream;
}
protected override void myOnFirstSubscriber() {
protected override void myOnFirstSubscriber()
{
_stream.onChange += new ChangeDelegate(_onstream_change);
}
protected override void myOnLastSubscriber() {
protected override void myOnLastSubscriber()
{
_stream.onChange -= new ChangeDelegate(_onstream_change);
}
private void _onstream_change(object o) {
private void _onstream_change(object o)
{
DataPacket dp = o as DataPacket;
DateTime now = DateTime.Now;
if (_table == null) {
if (_table == null)
{
IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
_table = new BlueTable(schema);
}
BlueTable added_rows = new BlueTable(_table.schema);
foreach (IBlueRow r in dp.addedRows) {
foreach (IBlueRow r in dp.addedRows)
{
MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
ar.setValue("timestamp", now);
ar.setValue("id", _id++);
@@ -42,4 +52,118 @@ namespace BCG {
}
}
}
public class TimeBucketStream : ViewStream, IStreamMapper
{
private ViewStream _src;
private readonly string _time_stamp_col;
private IBlueTableSchema _schema;
private int _bucket_size_seconds;
public TimeBucketStream(ViewStream input, string time_stamp_col, int bucket_size_seconds)
{
_time_stamp_col = time_stamp_col;
_bucket_size_seconds = bucket_size_seconds;
var vs = new MapStream(input, this);
mySetRealSource(vs);
}
#region IStreamMapper Members
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema)
{
IBlueTableSchema rv = input_schema;
//rv = BlueDataUtility.remove_columns(rv, _time_stamp_col);
rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int));
return rv;
}
#endregion
#region IDynamicStreamMapper Members
public IBlueRow map(IBlueRow row)
{
DateTime ts = row.get<DateTime>(_time_stamp_col);
TimeSpan time_of_day = (ts - ts.Date);
int seconds = Convert.ToInt32(time_of_day.TotalSeconds);
double buckets = seconds / _bucket_size_seconds;
int bucket_number = Convert.ToInt32(Math.Floor(buckets));
MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
rv.setValue("bucket_number", bucket_number);
return rv;
}
#endregion
}
// public class ThrottleStream : ViewStream, IWorker
// {
// private ViewStreamDatabase _vsd;
// private ViewStream _stream;
// private int _throttle_ms;
// private TimeSpan _min_fire_period;
// private DateTime _last_Fire_time = DateTime.MinValue;
// private DataPacket _unfired_dp;
// public ThrottleStream(ViewStreamDatabase vsd, ViewStream base_stream, int throttle_ms)
// {
// _vsd = vsd;
// _stream = base_stream;
// _throttle_ms = throttle_ms;
// _min_fire_period = TimeSpan.FromMilliSeconds(throttle_ms);
// }
// protected override void myOnFirstSubscriber()
// {
// _vsd.addWorker(this);
// _stream.onChange += new ChangeDelegate(_onstream_change);
// }
// protected override void myOnLastSubscriber()
// {
// _vsd.removeWorker(this);
// _stream.onChange -= new ChangeDelegate(_onstream_change);
// }
// private void _onstream_change(object obj)
// {
// DataPacket dp = obj as DataPacket;
// if (DateTime.Now - _last_Fire_time >= _min_fire_period)
// {
// // ok to fire here
// }
// else
// {
// _last_cached_state = dp.newRows;
// }
// _possibly_fire(dp.newRows);
// }
// public void doWork()
// {
// if (_unfired_dp != null)
// {
// }
// }
// private void _possibly_fire()
// {
// if (_unfired_dp != null)
// {
// if (DateTime.Now - _last_fire_time > _min_fire_period)
// {
// DataPacket dp = BlueDataUtility.build_dp_inc(_unfired_dp.newRows, myLastValue);
// _unfired_dp = null;
// _last_Fire_time = DateTime.Now;
// myFireEvent(dp);
// }
// }
// }
// }
}
@@ -6,7 +6,7 @@ using BCG.Utilities;
namespace BCG
{
[DataStreamUserRespresentation("csvSaver")]
[DataStreamUserRespresentation("CSVSaver")]
public class CSVSaverStream : ViewStream, ISideEffectStream
{
private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
Whitespace-only changes.
@@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Collections;
namespace BCG
{
[DataStreamUserRepresentation("timeBucket")]
public class TimeBucketStream : ViewStream, IStreamMapper
{
private ViewStream _src;
private readonly string _time_stamp_col;
private IBlueTableSchema _schema;
private uint _bucket_size_seconds;
public TimeBucketStream(ViewStream input, string time_stamp_col, uint bucket_size_seconds)
{
_time_stamp_col = time_stamp_col;
_bucket_size_seconds = bucket_size_seconds;
}
#region IStreamMapper Members
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema)
{
IBlueTableSchema rv = input_schema;
//rv = BlueDataUtility.remove_columns(rv, _time_stamp_col);
rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int));
return rv;
}
#endregion
#region IDynamicStreamMapper Members
public IBlueRow map(IBlueRow row)
{
DateTime ts = row.get<DateTime>(_time_stamp_col);
TimeSpan time_of_day = (ts - ts.Date)
int seconds = Convert.ToInt32(time_of_day.TotalSeconds);
int buckets = seconds / _bucket_size_seconds;
int bucket_number = Math.Floor(buckets);
MutableBlueRow rv = new MutableBlueRow(row, getOutputSchema(row.schema))
rv.setValue("bucket_number", bucket_number);
}
#endregion
}
}
@@ -1,10 +1,9 @@
using System;
using BCG.Utilities.General;
namespace BCG
{
public abstract class TimeUnitStream : ViewStream, IWorker
{
namespace BCG {
public abstract class TimeUnitStream : ViewStream, IWorker {
protected TimeSpan myOffset;
protected DateTime myConstructionTime;
protected int mySizeOfStream;
@@ -13,87 +12,80 @@ namespace BCG
protected ViewStreamDatabase myDb;
protected int myId = 0;
public TimeUnitStream (DateTime zeroTime, TimeSpan offSet, int sizeOfStream, ViewStreamDatabase db)
{
public TimeUnitStream(DateTime zeroTime, TimeSpan offSet, int sizeOfStream, ViewStreamDatabase db) {
myConstructionTime = DateTime.Now;
myZeroTime = zeroTime;
myOffset = offSet;
mySizeOfStream = sizeOfStream;
myDb = db;
myCollection = new CircularRowCollection (
new BlueTableSchema ("Time",
new ColumnInfo ("id", 0, typeof(Int32)), new ColumnInfo ("time", 1, typeof(DateTime))),
myCollection = new CircularRowCollection(
new BlueTableSchema("Time",
new ColumnInfo("id", 0, typeof(Int32)), new ColumnInfo("time", 1, typeof(DateTime))),
mySizeOfStream);
}
}
#region ViewStream overriden methods
protected override void myOnFirstSubscriber()
{
myDb.addWorker (this);
myInit ();
protected override void myOnFirstSubscriber() {
myDb.addWorker(this);
myInit();
}
protected override void myOnLastSubscriber()
{
myDb.removeWorker (this);
protected override void myOnLastSubscriber() {
myDb.removeWorker(this);
}
#endregion
protected virtual void myInit ()
{
protected virtual void myInit() {
int periods = (int)
(DateTime.Now - myConstructionTime).TotalMilliseconds / (int) myOffset.TotalMilliseconds;
(DateTime.Now - myConstructionTime).TotalMilliseconds / (int)myOffset.TotalMilliseconds;
DateTime dt = myGetLatestRow() == null ? myRoundTime(myZeroTime) :
myRoundTime (myZeroTime + new TimeSpan(myOffset.Ticks * periods));
DateTime dt = myGetLatestRow() == null ? myRoundTime(myZeroTime) :
myRoundTime(myZeroTime + new TimeSpan(myOffset.Ticks * periods));
if (myGetLatestRow() == null ||
!myGetLatestRow()["time"].Equals(dt))
{
IBlueRow ar = new BlueRow (myCollection.schema,
!myGetLatestRow()["time"].Equals(dt)) {
IBlueRow ar = new BlueRow(myCollection.schema,
myId++, dt);
IBlueRow dr = myCollection.addRowWithReturn (ar);
myFireEvent (new DataPacket (myCollection,
myConvertToCollection (ar),
myConvertToCollection (dr)));
IBlueRow dr = myCollection.addRowWithReturn(ar);
if (n_updates <= 3000)
myFireEvent(new DataPacket(myCollection,
myConvertToCollection(ar),
myConvertToCollection(dr)));
}
}
#region IWorker methods
public void doWork ()
{
myInit ();
public void doWork() {
myInit();
}
#endregion
protected abstract DateTime myRoundTime (DateTime dt);
protected abstract DateTime myRoundTime(DateTime dt);
#region Private Helper methods
private IBlueRow myGetLatestRow ()
{
private IBlueRow myGetLatestRow() {
return myCollection.Count == 0 ?
null :
BlueDataUtility.extractRow (myCollection, myCollection.Count - 1);
BlueDataUtility.extractRow(myCollection, myCollection.Count - 1);
}
private IBlueRowCollection myConvertToCollection (IBlueRow r)
{
BlueTable retVal = new BlueTable (myCollection.schema);
private IBlueRowCollection myConvertToCollection(IBlueRow r) {
BlueTable retVal = new BlueTable(myCollection.schema);
if (r != null)
retVal.addRow (r);
retVal.addRow(r);
return retVal;
}
#endregion
}
}
}
@@ -1,6 +1,8 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq.Expressions;
using System.Xml.Serialization;
using BCG.Utilities;
@@ -53,7 +55,12 @@ namespace BCG {
}
public virtual void createTable(IBlueRowCollection newTable) {
bts.Log.Gen.info($"PubTableBroker.createTable table: {newTable.schema.tableName}");
string log_msg = $"PubTableBroker.createTable table: {newTable.schema.tableName}\n";
foreach (IBlueRow r in newTable) {
log_msg += $"id: {r["id"]}";
}
bts.Log.Gen.info(log_msg);
_check_schema_consistency(newTable.schema);
myCheckTransaction();
IPubTablePublisher table_publisher = null;
var name_key = newTable.schema.tableName.ToLower();
@@ -71,6 +78,9 @@ namespace BCG {
/// </summary>
/// <param name="r"></param>
public virtual void addRow(IBlueRow r) {
string log_msg = $"PubTableBroker.addRow table: {r.schema.tableName} id: {r["id"]}";
bts.Log.Gen.info(log_msg);
_check_schema_consistency(r.schema);
myCheckTransaction();
IPubTablePublisher table_publisher = null;
var name_key = r.schema.tableName.ToLower();
@@ -86,6 +96,9 @@ namespace BCG {
/// </summary>
/// <param name="r"></param>
public virtual void deleteRow(IBlueRow r) {
string log_msg = $"PubTableBroker.deleteRow table: {r.schema.tableName} id: {r["id"]}";
bts.Log.Gen.info(log_msg);
_check_schema_consistency(r.schema);
myCheckTransaction();
IPubTablePublisher table_publisher = null;
var name_key = r.schema.tableName.ToLower();
@@ -101,6 +114,9 @@ namespace BCG {
/// </summary>
/// <param name="r"></param>
public virtual void changeRow(IBlueRow r) {
string log_msg = $"PubTableBroker.changeRow table: {r.schema.tableName} id: {r["id"]}";
bts.Log.Gen.info(log_msg);
_check_schema_consistency(r.schema);
myCheckTransaction();
IPubTablePublisher table_publisher = null;
var name_key = r.schema.tableName.ToLower();
@@ -118,7 +134,16 @@ namespace BCG {
public virtual void makeBatchChanges(IBlueRowCollection newRows,
IBlueRowCollection deletedRows,
IBlueRowCollection addedRows) {
myCheckTransaction();
string log_msg = $"PubTableBroker.makeBatchChanges {deletedRows.Count} {addedRows.Count} table: {newRows.schema.tableName} deleted_rows: ";
foreach (IBlueRow r in deletedRows)
log_msg += $"id: {r["id"]} ";
log_msg += "addedRows: ";
foreach (IBlueRow r in addedRows)
log_msg += $"id: {r["id"]}";
bts.Log.Gen.info(log_msg);
_check_schema_consistency(newRows.schema);
_check_schema_consistency(deletedRows.schema);
_check_schema_consistency(addedRows.schema);
IPubTablePublisher table_publisher = null;
var name_key = newRows.schema.tableName.ToLower();
if (!_publisher_map.TryGetValue(name_key, out table_publisher))
@@ -144,8 +169,16 @@ namespace BCG {
}
protected virtual IPubTablePublisher _create_pubtable_publisher(string tag) {
bts.Log.Gen.info($"PubTableBroker._create_pubtable_publisher tag: {tag}");
return new PubTablePublisher(tag, _publisher_core);
}
private void _check_schema_consistency(IBlueTableSchema schema_arg) {
IBlueTableSchema schema;
if (_schema_map.TryGetValue(schema_arg.tableName, out schema)) {
bts.Log.assert(BlueDataUtility.compareSchemas(schema, schema_arg));
}
}
}
}
@@ -14,7 +14,7 @@ namespace BCG {
/// is not uniquely chosen)
public ViewPublisher(string name, PubTableBroker br, ViewStream vs) {
myName = name;
myView = vs;//String.IsNullOrEmpty(name) ? vs : new ProjectStream(vs, "!", myName);
myView = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name);
myBroker = br;
myFirstTime = true;
myView.onChange += new DataStream.ChangeDelegate(myChangeHandler);
File diff suppressed because it is too large. Load diff
@@ -5,14 +5,12 @@ using System.Data;
//using BCG.Utilities;
using System.Linq;
namespace BCG
{
namespace BCG {
/// <summary>
/// Anologue of the dotNet DataTable.
/// </summary>
[Serializable]
public class BlueTable : IBlueRowCollection
{
public class BlueTable : IBlueRowCollection {
/// <summary>
/// Hashtable indexed on "id" to speed up removes
/// </summary>
@@ -29,22 +27,19 @@ namespace BCG
/// all the data over
/// </summary>
/// <param name="table"></param>
public BlueTable(DataTable table)
{
public BlueTable(DataTable table) {
ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
int i = 0;
foreach (DataColumn col in table.Columns)
cis[i] = new ColumnInfo(col.ColumnName, i++, col.DataType);
mySchema = new BlueTableSchema(cis, table.TableName);
foreach (DataRow row in table.Rows)
{
foreach (DataRow row in table.Rows) {
IBlueRow newRow = new BlueRow(mySchema, row.ItemArray);
myIdIndex[newRow["id"]] = newRow;
}
}
public BlueTable(IBlueRowCollection table)
{
public BlueTable(IBlueRowCollection table) {
mySchema = table.schema;
foreach (IBlueRow row in table)
addRow(row);
@@ -52,12 +47,10 @@ namespace BCG
public BlueTable(string tableName,
IEnumerable objects,
params CMap[] columnMappers)
{
params CMap[] columnMappers) {
// extract schema from mappers
ColumnInfo[] cols = new ColumnInfo[columnMappers.Length];
for (int i = 0; i < columnMappers.Length; i++)
{
for (int i = 0; i < columnMappers.Length; i++) {
cols[i] = new ColumnInfo(
columnMappers[i].getColumnName(), i,
columnMappers[i].getColumnType());
@@ -66,8 +59,7 @@ namespace BCG
// load rows
int rowNumber = 0;
foreach (object rowObject in objects)
{
foreach (object rowObject in objects) {
object[] values = new object[columnMappers.Length];
for (int cn = 0; cn < columnMappers.Length; cn++)
values[cn] = columnMappers[cn].getColumnValue(rowNumber, rowObject);
@@ -87,10 +79,8 @@ namespace BCG
/// <summary>
/// Returns the row by the index, interface property
/// </summary>
public virtual IBlueRow this[int id]
{
get
{
public virtual IBlueRow this[int id] {
get {
object row = myIdIndex[id];
if (row == null)
throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
@@ -105,8 +95,7 @@ namespace BCG
public virtual bool containsID(int id) { return myIdIndex[id] != null; }
public virtual bool tryGetValue(int id, out IBlueRow row)
{
public virtual bool tryGetValue(int id, out IBlueRow row) {
row = (IBlueRow)myIdIndex[id];
return (row != null);
}
@@ -114,8 +103,7 @@ namespace BCG
#endregion
#region BlueTable members
public virtual void clear()
{
public virtual void clear() {
myIdIndex.Clear();
}
@@ -123,8 +111,7 @@ namespace BCG
/// If a row with the same id exists, it replaces it
/// </summary>
/// <param name="r"></param>
public virtual void forceAddRow(IBlueRow r)
{
public virtual void forceAddRow(IBlueRow r) {
myIdIndex[r["id"]] = r;
}
@@ -132,43 +119,41 @@ namespace BCG
/// Adds a new row to the table.
/// </summary>
/// <param name="row"></param>
public virtual void addRow(IBlueRow row)
{
public virtual void addRow(IBlueRow row) {
object id = row["id"];
if (id == null || id == DBNull.Value)
throw new BlueDataException("The ID of the row being added is null/DNBull");
try
{
try {
myIdIndex.Add(id, row);
}
catch (ArgumentException)
{
throw new BlueDataException($"ID {id} already exists in the table: {mySchema.tableName} , duplication of field ID not allowed");
} catch (ArgumentException) {
throw new BlueDataException($"ID: {id} already exists in the table: {mySchema.tableName}");
}
}
public virtual void addRow(DataRow row)
{
public virtual void addRow(DataRow row) {
object val = row["id"];
if (val == null || val == DBNull.Value)
throw new BlueDataException("The ID of the row being added is null/DNBull");
if (myIdIndex[val] != null)
throw new BlueDataException("ID already exists in the table, duplication of field ID not allowed");
if (myIdIndex[val] != null) {
string err_msg = $"ID: {val} already exists in the table: {mySchema.tableName} duplication of field ID not allowed";
// temp TODOds remove it
bts.Log.assert(false, err_msg);
throw new BlueDataException(err_msg);
}
IBlueRow newRow = new BlueRow(mySchema, row);
myIdIndex[val] = newRow;
}
public virtual void addRows(IBlueRowCollection rows)
{
public virtual void addRows(IBlueRowCollection rows) {
foreach (IBlueRow nr in rows)
addRow(nr);
}
public virtual void addRows(IEnumerable rows)
{
public virtual void addRows(IEnumerable rows) {
foreach (IBlueRow nr in rows)
addRow(nr);
}
@@ -185,37 +170,32 @@ namespace BCG
/// Removes a row specified by the id. The index helps in making this operation O(1)
/// </summary>
/// <param name="id"></param>
public virtual bool deleteRow(int id)
{
public virtual bool deleteRow(int id) {
if (myIdIndex[id] == null)
return false;
myIdIndex.Remove(id);
return true;
}
public virtual bool deleteRows(IBlueRowCollection rows)
{
public virtual bool deleteRows(IBlueRowCollection rows) {
bool retVal = true;
foreach (IBlueRow r in rows)
retVal &= deleteRow(r);
return retVal;
}
public virtual bool deleteRows(IEnumerable rows)
{
public virtual bool deleteRows(IEnumerable rows) {
bool retVal = true;
foreach (IBlueRow r in rows)
retVal &= deleteRow(r);
return retVal;
}
public virtual void changeRow(IBlueRow r)
{
public virtual void changeRow(IBlueRow r) {
myIdIndex[r["id"]] = r;
}
public virtual void changeRows(IBlueRowCollection rows)
{
public virtual void changeRows(IBlueRowCollection rows) {
foreach (IBlueRow r in rows)
changeRow(r);
}
@@ -42,7 +42,7 @@ namespace BCG
Delegate.Combine (myOnUpdateDelegate, value) as TableUpdateDelegate;
if (myStateTable != null)
value (myStateTable, myStateTable, new BlueTable (myStateTable.schema));
}
}
remove
{
// BlueEventLog.logEvent ("TableSubscriber being Unsubscribed for tag :" + myTag, BlueEventLevel.Information);
@@ -64,6 +64,7 @@ namespace BCG
public void update(Stream[] streams)
{
bts.Log.Gen.info($"TableAccumulator.update tag: {myTag} n_streams: {streams.Length} stream[0].len: {streams[0].Length - streams[0].Position}");
//checking the condition as a subscriber might unsubscribe in a callback and there might be more than one
//data element for that tag in the ViewStreamDatabase
if (myOnUpdateDelegate != null)
@@ -3,16 +3,15 @@ using System.IO;
using System.Collections;
using BCG.Utilities;
namespace BCG
{
namespace BCG {
/// <summary>
/// Controls (ie creates dynamically) an IClientDeserializer and then passes data to it.
/// </summary>
public class ClientDeserializationController : IClientDeserializer
{
public class ClientDeserializationController : IClientDeserializer {
private IClientDeserializer myDeserializer;
public ClientDeserializationController () {}
private int _n_desers = 0;
public ClientDeserializationController() { }
#region IClientDeserializer Members
@@ -23,42 +22,41 @@ namespace BCG
/// <param name="streams"></param>
/// <param name="theOps"></param>
/// <param name="theRows"></param>
public void deserialize (Stream[] streams, out byte[] theOps, out Array theRows)
{
if (myDeserializer != null)
{
myDeserializer.deserialize (streams, out theOps, out theRows);
}
else
{
Stream totalStream = UtilityMethods.getAggregatedStream (streams);
public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) {
bts.Log.Gen.info($"ClientDeserializationController.deserialize len: {streams[0].Length - streams[0].Position}");
if (myDeserializer != null) {
myDeserializer.deserialize(streams, out theOps, out theRows);
} else {
Stream totalStream = UtilityMethods.getAggregatedStream(streams);
long initialPos = totalStream.Position;
BinaryReader br = new BinaryReader (totalStream);
BinaryReader br = new BinaryReader(totalStream);
byte op;
try
{
op = br.ReadByte ();
}
catch (EndOfStreamException)
{
try {
op = br.ReadByte();
} catch (EndOfStreamException) {
op = Byte.MaxValue;
}
if (op == StreamOpCode.create)
{
br.ReadString (); //pubsubclientdeserializer
string deserName = br.ReadString ();
myDeserializer = Activator.CreateInstance (Type.GetType (deserName))
if (op == StreamOpCode.create) {
br.ReadString(); //pubsubclientdeserializer
string deserName = br.ReadString();
myDeserializer = Activator.CreateInstance(Type.GetType(deserName))
as IClientDeserializer;
totalStream.Position = initialPos;
myDeserializer.deserialize (new Stream[] {totalStream}, out theOps, out theRows);
}
else
{
myDeserializer.deserialize(new Stream[] { totalStream }, out theOps, out theRows);
} else {
// BlueEventLog.logEvent ("Received a non create before a create", BlueEventLevel.Warning);
theOps = new byte[]{};
theRows = new object[]{};
theOps = new byte[] { };
theRows = new object[] { };
}
}
var msgs = StreamIOMessage.assemble_msgs(theOps, theRows);
int i = 0;
foreach (var m in msgs) {
bts.Log.Gen.info($"ClientDeserializationController n_desers: {_n_desers} msg no: {i++}: {m}");
}
_n_desers++;
// log it here
}
@@ -5,61 +5,61 @@ using System.IO;
using BCG.Utilities;
//using BCG.Utilities.BlueConfiguration;
namespace BCG
{
namespace BCG {
/// <summary>
/// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process
/// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations)
/// </summary>
internal class ClientUniversalRowDeserializer : IClientRowDeserializer
{
internal class ClientUniversalRowDeserializer : IClientRowDeserializer {
private IBlueTableSchema mySchema;
private int myNullMapLength = 0;
private PrimitiveDeserializationHelper myDeserializationHelper;
private BitArray myNullMap;
private int[] mySerIndexToSchemaIndexMap;
public ClientUniversalRowDeserializer (IBlueTableSchema schema)
{
myDeserializationHelper = new PrimitiveDeserializationHelper (mySchema = schema);
myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8)));
myInitReorderSchema ();
public ClientUniversalRowDeserializer(IBlueTableSchema schema) {
myDeserializationHelper = new PrimitiveDeserializationHelper(mySchema = schema);
myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length) / 8)));
myInitReorderSchema();
}
#region Schema Reordering initialization
private void myInitReorderSchema ()
{
int idColIndex = mySchema["id"].colNo;
int[] temp = new int [mySchema.columnInfo.Length];
foreach (ColumnInfo ci in mySchema.columnInfo)
{
temp[ci.colNo] =
ci.colNo == idColIndex ? 0 :
ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo;
private void myInitReorderSchema() {
try {
int idColIndex = mySchema["id"].colNo;
int[] temp = new int[mySchema.columnInfo.Length];
foreach (ColumnInfo ci in mySchema.columnInfo) {
temp[ci.colNo] =
ci.colNo == idColIndex ? 0 :
ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo;
}
mySerIndexToSchemaIndexMap = new int[temp.Length];
for (int i = 0; i < temp.Length; i++)
mySerIndexToSchemaIndexMap[temp[i]] = i;
} catch (Exception ex) {
string schema_name = mySchema == null ? "NULL" : mySchema.tableName;
string err_msg = $"ClientUniversalRowDeserializer.myInitReorderSchema table: {schema_name} ex: {ex}";
bts.Log.Gen.error(err_msg);
throw new ApplicationException(err_msg);
}
mySerIndexToSchemaIndexMap = new int[temp.Length];
for (int i = 0; i < temp.Length; i ++)
mySerIndexToSchemaIndexMap [temp[i]] = i;
}
#endregion
#region IClientRowDeserializer Members
public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
{
br.ReadInt32 (); //length information for the row not needed
myNullMap = new BitArray(br.ReadBytes (myNullMapLength));
object[] values = new object [mySchema.columnNames.Length];
for (int i = 0; i < values.Length; i++)
{
public void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) {
br.ReadInt32(); //length information for the row not needed
myNullMap = new BitArray(br.ReadBytes(myNullMapLength));
object[] values = new object[mySchema.columnNames.Length];
for (int i = 0; i < values.Length; i++) {
int colNo = mySerIndexToSchemaIndexMap[i];
values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn (colNo, br);
values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn(colNo, br);
}
r = new BlueRow(mySchema, values);
r = new BlueRow(mySchema, values);
}
@@ -1,22 +1,149 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace BCG {
public class StreamIOMessage {
public enum StreamOp {
Create = 0,
Add = 1, //adds a single row
Remove = 2, //removes a single row
Change = 3, //changes a single row
MultipleRemoveAdd = 4,//removes and adds multiple rows
MultipleChange = 5 //changes multiples rows
}
public class CreateMessage {
public IBlueRowCollection rows;
public override string ToString() {
string rv = "Create message ";
if (rows == null) {
rv += " rows: null";
return rv;
}
foreach (IBlueRow r in rows) {
rv += $"id: {r["id"]} ";
}
return rv;
}
}
public class AddMessage {
public IBlueRow added_row;
}
public class RemoveMessage {
public IBlueRow removed_row;
}
public class ChangeMessage {
public IBlueRow changed_row;
}
public class MultipleRemoveAddMessage {
public IBlueRowCollection removed;
public IBlueRowCollection added;
public override string ToString() {
string rv = "MultipleRemoveAdd removed: ";
foreach (IBlueRow r in removed) {
rv += $"id: {r["id"]} ";
}
rv += "added: ";
foreach (IBlueRow r in added) {
rv += $"id: {r["id"]} ";
}
return rv;
}
}
public class MultipleChangeMessage {
public IBlueRowCollection changed;
}
public StreamOp op;
public CreateMessage create_message = null;
public AddMessage add_message = null;
public RemoveMessage remove_message = null;
public ChangeMessage change_message = null;
public MultipleRemoveAddMessage multiple_remove_add_message = null;
public MultipleChangeMessage multiple_change_message = null;
public StreamIOMessage(byte op_, object data) {
op = (StreamIOMessage.StreamOp)(op_);
switch (op) {
case StreamIOMessage.StreamOp.Create:
IBlueRowCollection state = null;
// TODO, not sure why it can be an array and by itself
if (data is IBlueRowCollection) state = data as IBlueRowCollection;
else if (data is IBlueRowCollection[]) state = (data as IBlueRowCollection[]).ElementAt(0);
create_message = new CreateMessage() { rows = state };
break;
case StreamIOMessage.StreamOp.Add:
bts.Log.assert(false);
break;
case StreamIOMessage.StreamOp.Remove:
bts.Log.assert(false);
break;
case StreamIOMessage.StreamOp.Change:
bts.Log.assert(false);
break;
case StreamIOMessage.StreamOp.MultipleRemoveAdd:
IBlueRowCollection[] row_args = data as IBlueRowCollection[];
multiple_remove_add_message = new MultipleRemoveAddMessage() { removed = row_args[0], added = row_args[1] };
break;
case StreamIOMessage.StreamOp.MultipleChange:
//IBlueRowCollection[] row_args = data as IBlueRowCollection[];
multiple_change_message = new MultipleChangeMessage() { changed = (data as IBlueRowCollection[])[0] };
break;
}
}
public override string ToString() {
switch (op) {
case StreamIOMessage.StreamOp.Create:
return create_message.ToString();
break;
case StreamIOMessage.StreamOp.Add:
bts.Log.assert(false);
return String.Empty;
break;
case StreamIOMessage.StreamOp.Remove:
bts.Log.assert(false);
return String.Empty;
break;
case StreamIOMessage.StreamOp.Change:
bts.Log.assert(false);
return String.Empty;
break;
case StreamIOMessage.StreamOp.MultipleRemoveAdd:
return multiple_remove_add_message.ToString();
break;
case StreamIOMessage.StreamOp.MultipleChange:
return multiple_change_message.ToString();
break;
}
bts.Log.assert(false);
return String.Empty;
}
public static IEnumerable<StreamIOMessage> assemble_msgs(byte[] ops, Array op_data) {
int i = 0;
List<StreamIOMessage> rv = new List<StreamIOMessage>();
foreach (byte op in ops) {
object data = op_data.GetValue(i++);
rv.Add(new StreamIOMessage(op, data));
}
return rv;
}
}
namespace BCG
{
/// <summary>
/// Performs deserialization inside the client app. The client gets either the complete data set or the
/// increments from PubSub and this interface makes sense out of them from a tabular/datastream stand point.
/// </summary>
public interface IClientDeserializer
{
public interface IClientDeserializer {
/// <summary>
/// deserializes the data received by the client from the pubsub
/// </summary>
/// <param name="serializedData">the data in byte format received from pubsub</param>
/// <param name="theOps">the operations that the data maps onto</param>
/// <param name="theRows">the data corresponding to the operations, is an array of IblueRowCollection[]</param>
void deserialize (Stream[] streams, out byte[] theOps, out Array theRows);
void deserialize(Stream[] streams, out byte[] theOps, out Array theRows);
}
}
@@ -2,13 +2,11 @@ using System;
using System.IO;
using System.Collections;
namespace BCG
{
namespace BCG {
/// <summary>
/// Deserializes one row. the current state is also provided as some deserialization schemes might use it.
/// </summary>
public interface IClientRowDeserializer
{
void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r);
public interface IClientRowDeserializer {
void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r);
}
}
@@ -16,7 +16,7 @@ namespace BCG {
public PubTablePublisher(string tag, PublisherCore publisherCore) {
_tag = tag;
bts.Log.Gen.info($"PubTablePublisher created for tag: {tag}");
bts.Log.Gen.info($"PubTablePublisher.cons created for tag: {tag}");
myPubCore = publisherCore;
mySerializer = new BlueDataSerializationController();
}
@@ -54,10 +54,14 @@ namespace BCG {
out data);
bts.Log.Gen.info($"PubTablePublisher.createTable tag: {_tag} rows: {rows.Count} data: {data.Length}");
myPublish(data, rows.schema.tableName);
// byte[] ops;
// Array rows_ds;
// _test_deserializer.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows_ds);
}
//private ClientDeserializationController _test_deserializer = new ClientDeserializationController();
public virtual void batchRemoveAdd(IBlueRowCollection delRows, IBlueRowCollection addRows) {
byte[] data;
byte[] data = null;
if (myIsRealChange(delRows, addRows))
batchChange(addRows);
else {
@@ -66,6 +70,9 @@ namespace BCG {
out data);
myPublish(data, delRows.schema.tableName);
}
// byte[] ops;
// Array rows_ds;
// _test_deserializer.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows_ds);
}
public virtual void batchChange(IBlueRowCollection rows) {
@@ -1,40 +1,84 @@
using System;
using mbts.config;
using bts.utils;
using System.Threading.Tasks;
using Bts.DataStreamExpressions;
using System.Collections.Generic;
using BCG;
bts.utils.BTSNativeMethods.provide_command_line(args, args.Length);
mbts.config.Configuration.initialize();
bts.Log.Gen.info($"Starting DSPublisher");
Console.WriteLine("Starting DSPublisher...");
// static async Task _run() {
// var login = "bts";
// var passwd = String.Empty;
// Console.WriteLine("Starting DSQA... ");
// // Log.notnull(login);
// // Log.notnull(passwd);
// // var service_ctx = bts.api.core.ServiceContext.create();
// await service_ctx.login(login, passwd);
// }
//Task.Run(async () => await _run());
_publish_stream();
void _publish_stream() {
BCG.ViewStream vs = new LiteralStream("x", "420.0");// new BBOStream();
//new BCG.MDStream();
ViewPublisher vpub = new ViewPublisher(String.Empty, new PubTableBroker(), vs);
vs.onChange += on_change;
static void init(string[] args)
{
Console.WriteLine("Hit key to continue");
Console.ReadLine();
Func<string, string, Task> login = async (l, p) =>
{
var login = l;
var passwd = p;
var service_ctx = bts.api.core.ServiceContext.create();
await service_ctx.login(login, passwd);
};
BTSNativeMethods.provide_command_line(args, args.Length);
Configuration.initialize();
Task.Run(async () => await login("bts", String.Empty));
}
while (true) {
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
ViewStreamDatabase.instance.doWork();
init(args);
_publish_streams();
static ViewStream build_stream(string stream_expression)
{
try
{
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv;
}
catch (Exception ex)
{
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
return null;
}
}
void on_change(object obj) {
DataPacket dp = obj as DataPacket;
dp.newRows.print_table((col) => { return 10; });
void _publish_streams()
{
PubTableBroker pb_broker = new PubTableBroker();
CloudStreamManager csm = new CloudStreamManager();
// get it from config
Dictionary<string, string> streams_to_publish = new Dictionary<string, string>() {
{"bbo","bbo()"},
{"instruments","instruments()"},
//{"timesales","timesales(10000)"},
{"books","books()"},
// {"seconds_1_10","second(1,10)"},
{"literal_420","literal(\"x\", \"420.0\")"},
// {"bbo_ts","timeSeries(bbo())"}
};
foreach (var kv in streams_to_publish)
{
var name = kv.Key;
var exp = kv.Value;
BCG.ViewStream vs = build_stream(exp);
ViewPublisher vpub = new ViewPublisher(name, pb_broker, vs);
vs.onChange += on_change;
}
while (true)
{
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
ViewStreamDatabase.instance.doWork();
csm.doWork();
}
}
void on_change(object obj)
{
bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
}
@@ -25,17 +25,19 @@ using System.Threading.Tasks;
namespace BCG {
public partial class MainForm : System.Windows.Forms.Form {
static async Task _run() {
var login = "bts";
var passwd = String.Empty;
Console.WriteLine("Starting DSQA... ");
// Log.notnull(login);
// Log.notnull(passwd);
var service_ctx = bts.api.core.ServiceContext.create();
await service_ctx.login(login, passwd);
}
namespace BCG
{
public partial class MainForm : System.Windows.Forms.Form
{
// static async Task _run() {
// var login = "bts";
// var passwd = String.Empty;
// Console.WriteLine("Starting DSQA... ");
// // Log.notnull(login);
// // Log.notnull(passwd);
// var service_ctx = bts.api.core.ServiceContext.create();
// await service_ctx.login(login, passwd);
// }
private bool myChangeStyle = false;
private PubSubRoute[] myRoutes = PubSub3Config.instance.getSubRouting();
private PersistWindowState myWindowState;
@@ -50,8 +52,9 @@ namespace BCG {
private string myKQ;
private TextWriter myConsoleOutput;
public MainForm() {
Task.Run(async () => await _run());
public MainForm()
{
//Task.Run(async () => await _run());
InitializeComponent();
// myKQ = ConfigurationRequester.getApplicationQualifier();
myKQ = "TODO";
@@ -65,17 +68,20 @@ namespace BCG {
this.Closing += new CancelEventHandler(myClosing);
}
public string query_text {
public string query_text
{
get { return myQueryTextBox.Text; }
set { myQueryTextBox.Text = value; }
}
private ViewStream myGetAllTagsStream() {
private ViewStream myGetAllTagsStream()
{
// TODO: Port PublishedTagsStream from edge (then we can uncomment impl below)
return new FilterStream(new PubSubServerStateStream(), "tag~='DS_'");
//return new LiteralStream("tag/String", "DS_Comedians;DS_Baldies");
}
private void mySetText() {
private void mySetText()
{
this.Text = "Data Stream Query Analyzer | Site: " + mySite + (string.IsNullOrEmpty(myKQ) ? string.Empty : " | Key Qualifier: " + myKQ);
myToolStripStatusLabelTime.Text = "Ready";
myToolStripStatusLabelRows.Text = string.Empty;
@@ -85,10 +91,12 @@ namespace BCG {
/// Gets the data in the form of a multilevel IDictionary
/// </summary>
/// <returns></returns>
private IDictionary myGetLibraryData() {
private IDictionary myGetLibraryData()
{
SortedList retVal = new SortedList();
string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames();
foreach (string pName in primitiveNames) {
foreach (string pName in primitiveNames)
{
string aName = PrimitiveManager.instance.getAssemblyName(pName);
ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName);
if (!retVal.Contains(aName))
@@ -103,12 +111,15 @@ namespace BCG {
}
private void myLoadAssemblies() {
private void myLoadAssemblies()
{
myTreeView.Nodes.Clear();
IDictionary libraryData = myGetLibraryData();
foreach (DictionaryEntry de in libraryData) {
foreach (DictionaryEntry de in libraryData)
{
TreeNode assemblyNode = new TreeNode(de.Key as String);
foreach (DictionaryEntry innerDe in (de.Value as IDictionary)) {
foreach (DictionaryEntry innerDe in (de.Value as IDictionary))
{
TreeNode primitiveNode = new TreeNode(innerDe.Key as string);
foreach (string con in (innerDe.Value as ICollection))
primitiveNode.Nodes.Add(con);
@@ -128,23 +139,27 @@ namespace BCG {
}*/
}
private void myHandleTableListChange(object o) {
private void myHandleTableListChange(object o)
{
IBlueRowCollection rows = (o as DataPacket).newRows;
myUpdateTableList();
}
#region Persistence Keys
public int textBoxHeight {
public int textBoxHeight
{
get { return myQueryTextBox.Height; }
set { myQueryTextBox.Height = value; }
}
#endregion
private void myTryExpression(string expStr) {
private void myTryExpression(string expStr)
{
myFreezeView(false); // force unfreeze the view
DataStreamExpression oldExp = myExp;
try {
try
{
var exp = DataStreamExpressionsInterpreter.Evaluate(expStr);
if (exp == null)
@@ -171,7 +186,9 @@ namespace BCG {
oldExp.getView(new Hashtable()).onChange -= new DataStream.ChangeDelegate(myCallback);
*/
//myQueryTextBox.Text = expStr;
} catch (Exception exc) {
}
catch (Exception exc)
{
Log.Ui.warn(exc.ToString());
myDataGrid.DataSource = null;
@@ -185,7 +202,8 @@ namespace BCG {
}
}
private string myExtractFromClass(TextReader r) {
private string myExtractFromClass(TextReader r)
{
string line = r.ReadLine();
while (line != null && line.IndexOf("*Expression*") == -1)
line = r.ReadLine();
@@ -198,7 +216,8 @@ namespace BCG {
string result = "";
line = r.ReadLine();
while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1) {
while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1)
{
if (line != "")
result += line.Substring(" ".Length) + "\n";
line = r.ReadLine();
@@ -209,17 +228,24 @@ namespace BCG {
return result;
}
public void loadFile(string fileName) {
try {
public void loadFile(string fileName)
{
try
{
TextReader r = new StreamReader(fileName);
try {
try
{
myQueryTextBox.Text = myExtractFromClass(r);
} finally {
}
finally
{
r.Close();
}
myLastFileName = fileName;
this.Text = fileName;
} catch (Exception exc) {
}
catch (Exception exc)
{
Log.Ui.warn(exc.ToString());
myDataGrid.DataSource = null;
@@ -230,7 +256,8 @@ namespace BCG {
}
private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code) {
private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code)
{
myCSharpText.Text = "using System;\n\nnamespace BCG\n{";
@@ -250,7 +277,8 @@ namespace BCG {
string lastVar = null;
string variableDeclarations = "";
foreach (string var in vars) {
foreach (string var in vars)
{
variableDeclarations += " ViewStream " + var + ";\n";
lastVar = var;
}
@@ -280,13 +308,16 @@ namespace BCG {
myCSharpText.Text += "}";
}
static DataGridViewColumn _column_factory(ColumnInfo ci) {
return new DataGridViewTextBoxColumn {
static DataGridViewColumn _column_factory(ColumnInfo ci)
{
return new DataGridViewTextBoxColumn
{
HeaderText = ci.colName,
Width = get_col_width(ci)
};
static int get_col_width(ColumnInfo ci) {
static int get_col_width(ColumnInfo ci)
{
const int CHAR_WIDTH = 10;
const int MIN_WIDTH = 50;
const int MIN_DATE_WIDTH = 200;
@@ -297,12 +328,14 @@ namespace BCG {
}
private HighPerformanceTimer _timer = new HighPerformanceTimer();
// this is the stream callback
private void myCallback(object o) {
private void myCallback(object o)
{
_timer.start();
DataPacket dp = o as DataPacket;
myCrono.stop();
Log.Ui.diag($"Received callback for {dp.schema.tableName}");
if (o == null) {
if (o == null)
{
System.Windows.Forms.MessageBox.Show(this, "Received a Null Data packet",
"Invalid Data", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
@@ -319,7 +352,8 @@ namespace BCG {
myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows);
var force = false;
if (myChangeStyle && force) {
if (myChangeStyle && force)
{
myChangeStyle = false;
myDataGrid.Columns.Clear();
myDataGrid.Columns.AddRange(Array.ConvertAll(dp.schema.columnInfo, _column_factory));
@@ -346,11 +380,15 @@ namespace BCG {
private DataStreamExpression myExp;
private void myTimer_Tick(object sender, System.EventArgs e) {
try {
private void myTimer_Tick(object sender, System.EventArgs e)
{
try
{
ViewStreamDatabase.instance.doWork();
MasterWorker.getInstance(false).doWork();
} catch (Exception exc) {
}
catch (Exception exc)
{
Log.Ui.warn(exc.ToString());
myDataGrid.DataSource = null;
@@ -361,22 +399,27 @@ namespace BCG {
}
}
class StringWrapper {
public StringWrapper(string s) {
class StringWrapper
{
public StringWrapper(string s)
{
myTableName = s;
}
public string tableName {
public string tableName
{
get { return myTableName; }
}
private string myTableName;
}
private void MainForm_Closed(object sender, System.EventArgs e) {
private void MainForm_Closed(object sender, System.EventArgs e)
{
if (myExp != null)
myExp.getView(new Dictionary<string, StatementExpression>(0)).onChange -= new DataStream.ChangeDelegate(myCallback);
}
private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) {
private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e)
{
myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
myQueryTextBox.SelectionColor = Color.Black;
@@ -384,22 +427,24 @@ namespace BCG {
myTryExpression(myQueryTextBox.Text);
}
private void myToolStripMenuItemExecSel_Click(object sender, EventArgs e) {
int selstart = myQueryTextBox.SelectionStart;
int sellength = myQueryTextBox.SelectionLength;
myQueryTextBox.SelectionStart = 0;
myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
myQueryTextBox.SelectionColor = Color.Black;
myQueryTextBox.SelectionStart = selstart;
myQueryTextBox.SelectionLength = sellength;
myQueryTextBox.SelectionColor = Color.Crimson;
myTryExpression(myQueryTextBox.SelectedText);
myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
myQueryTextBox.SelectionLength = 0;
myQueryTextBox.SelectionColor = Color.Black;
private void myToolStripMenuItemExecSel_Click(object sender, EventArgs e)
{
// int selstart = myQueryTextBox.SelectionStart;
// int sellength = myQueryTextBox.SelectionLength;
// myQueryTextBox.SelectionStart = 0;
// myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
// myQueryTextBox.SelectionColor = Color.White;
// myQueryTextBox.SelectionStart = selstart;
// myQueryTextBox.SelectionLength = sellength;
// myQueryTextBox.SelectionColor = Color.Crimson;
// myTryExpression(myQueryTextBox.SelectedText);
// myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
// myQueryTextBox.SelectionLength = 0;
// myQueryTextBox.SelectionColor = Color.White;
}
private void myMenuItemOpenClick(object sender, System.EventArgs e) {
private void myMenuItemOpenClick(object sender, System.EventArgs e)
{
OpenFileDialog od = new OpenFileDialog();
od.Filter = "C# class (*.cs)|*.cs";
@@ -407,21 +452,24 @@ namespace BCG {
loadFile(od.FileName);
}
private void mySave(string fileName) {
private void mySave(string fileName)
{
using (TextWriter w = new StreamWriter(fileName))
w.Write(myCSharpText.Text);
myLastFileName = fileName;
this.Text = fileName;
}
private void myMenuItemSaveClick(object sender, System.EventArgs e) {
private void myMenuItemSaveClick(object sender, System.EventArgs e)
{
if (myLastFileName == null)
myMenuItemSaveAsClick(sender, e);
else
mySave(myLastFileName);
}
private void myMenuItemSaveAsClick(object sender, System.EventArgs e) {
private void myMenuItemSaveAsClick(object sender, System.EventArgs e)
{
SaveFileDialog sd = new SaveFileDialog();
if (myLastFileName != null)
sd.FileName = myLastFileName;
@@ -432,47 +480,56 @@ namespace BCG {
mySave(sd.FileName);
}
private void myTableListBox_DoubleClick(object sender, System.EventArgs e) {
if (myTableListBox.SelectedItem != null) {
private void myTableListBox_DoubleClick(object sender, System.EventArgs e)
{
if (myTableListBox.SelectedItem != null)
{
string tableName = myTableListBox.SelectedItem.ToString();
myQueryTextBox.Text = myQueryTextBox.Text == "" ? tableName : myQueryTextBox.Text + "\n" + tableName;
myTryExpression(myQueryTextBox.Text);
}
}
private void myDataGrid_MouseEnter(object sender, System.EventArgs e) {
private void myDataGrid_MouseEnter(object sender, System.EventArgs e)
{
//myDataGrid.TableStyles[0].GridLineColor = Color.LightSteelBlue;
// if (myDataGrid.TableStyles.Count > 0)
// myDataGrid.TableStyles[0].HeaderBackColor = Color.LightSteelBlue;
}
private void myDataGrid_MouseLeave(object sender, System.EventArgs e) {
private void myDataGrid_MouseLeave(object sender, System.EventArgs e)
{
//myDataGrid.TableStyles[0].GridLineColor = Color.Gainsboro;
// if (myDataGrid.TableStyles.Count > 0)
// myDataGrid.TableStyles[0].HeaderBackColor = Color.Gainsboro;
}
private void myCSharpText_MouseEnter(object sender, System.EventArgs e) {
private void myCSharpText_MouseEnter(object sender, System.EventArgs e)
{
//myCSharpText.BackColor = Color.Gainsboro;
myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Bold);
}
private void myCSharpText_MouseLeave(object sender, System.EventArgs e) {
private void myCSharpText_MouseLeave(object sender, System.EventArgs e)
{
//myCSharpText.BackColor = Color.White;
myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Regular);
}
private void myTableListBox_MouseEnter(object sender, System.EventArgs e) {
private void myTableListBox_MouseEnter(object sender, System.EventArgs e)
{
//myTableListBox.BackColor = Color.Gainsboro;
myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Bold);
}
private void myTableListBox_MouseLeave(object sender, System.EventArgs e) {
private void myTableListBox_MouseLeave(object sender, System.EventArgs e)
{
//myTableListBox.BackColor = Color.White;
myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Regular);
}
private void myLoad(object sender, EventArgs e) {
private void myLoad(object sender, EventArgs e)
{
myFreezeView(false);
myToolStripMenuItemSite.DropDownItems.Clear();
@@ -496,7 +553,9 @@ namespace BCG {
{
var ex = new Win32Exception(); // same as new Win32Exception(Marshal.GetLastWin32Error())
Console.WriteLine(ex.Message);
} else {
}
else
{
Console.WriteLine("Set stdout worked.");
}
@@ -504,7 +563,9 @@ namespace BCG {
{
var ex = new Win32Exception(); // same as new Win32Exception(Marshal.GetLastWin32Error())
Console.WriteLine(ex.Message);
} else {
}
else
{
Console.WriteLine("Set stderr worked.");
}
Console.WriteLine("Redirecting console ouput to this window...");
@@ -520,10 +581,12 @@ namespace BCG {
//_library = new LibraryForm(this);
}
private void myMenuItemSiteClick(object sender, System.EventArgs e) {
private void myMenuItemSiteClick(object sender, System.EventArgs e)
{
ToolStripMenuItem mi = (ToolStripMenuItem)sender;
if (mi.Text != mySite) {
if (mi.Text != mySite)
{
mySite = mi.Text;
foreach (var m in mySiteItems)
m.Checked = false;
@@ -544,63 +607,78 @@ namespace BCG {
}
}
private void myClosing(object sender, CancelEventArgs e) {
private void myClosing(object sender, CancelEventArgs e)
{
}
private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e) {
if (myTreeView.SelectedNode != null) {
if (myTreeView.SelectedNode.LastNode == null) {
private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e)
{
if (myTreeView.SelectedNode != null)
{
if (myTreeView.SelectedNode.LastNode == null)
{
string sn = myTreeView.SelectedNode.Parent.Text + "()";
myQueryTextBox.Text = myQueryTextBox.Text == "" ? sn : myQueryTextBox.Text + "\n" + sn;
}
}
}
private void myToolStripMenuItemExit_Click(object sender, EventArgs e) {
private void myToolStripMenuItemExit_Click(object sender, EventArgs e)
{
Close();
}
private void myToolStripMenuItemFreeze_Click(object sender, EventArgs e) {
private void myToolStripMenuItemFreeze_Click(object sender, EventArgs e)
{
myFreezeView(myTimer.Enabled);
}
private void myFreezeView(bool freeze) {
private void myFreezeView(bool freeze)
{
myTimer.Enabled = !freeze;
myToolStripMenuItemFreeze.Checked = freeze;
myToolStripMenuItemStep.Enabled = freeze;
}
private void myToolStripMenuItemStep_Click(object sender, EventArgs e) {
private void myToolStripMenuItemStep_Click(object sender, EventArgs e)
{
ViewStreamDatabase.instance.doWork();
}
private void myToolStripMenuItemLoadAss_Click(object sender, EventArgs e) {
private void myToolStripMenuItemLoadAss_Click(object sender, EventArgs e)
{
OpenFileDialog ofd = new OpenFileDialog();
ofd.ShowReadOnly = true;
DialogResult dr = ofd.ShowDialog(this);
if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK) {
if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK)
{
PrimitiveManager.instance.loadAssembly(ofd.FileName);
myLoadAssemblies();
}
}
private void myToolStripMenuItemTest_Click(object sender, EventArgs e) {
if (myCurrentStream != null) {
private void myToolStripMenuItemTest_Click(object sender, EventArgs e)
{
if (myCurrentStream != null)
{
myCurrentStream.onChange -= new DataStream.ChangeDelegate(myCallback);
myCurrentStream.onChange += new DataStream.ChangeDelegate(myCallback);
}
}
private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e) {
private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e)
{
myConsoleRichTextBox.Clear();
Console.WriteLine("Redirecting console ouput to this window...");
}
private void myTableFilterBox_TextChanged(object sender, EventArgs e) {
private void myTableFilterBox_TextChanged(object sender, EventArgs e)
{
myUpdateTableList();
}
private void myUpdateTableList() {
private void myUpdateTableList()
{
IBlueRowCollection rows = myAllTagsStream.rows;
if (rows == null)
return;
@@ -615,12 +693,15 @@ namespace BCG {
myTableListBox.Items.Clear();
SortedList<string, bool> sl = new SortedList<string, bool>();
if (string.IsNullOrEmpty(myTableFilterBox.Text))
foreach (IBlueRow r in rows) {
foreach (IBlueRow r in rows)
{
sl[r["tag"].ToString().Substring(PubStreamPrefixer.prefix("").Length)] = true;
}
else {
else
{
Regex tableRegex = new Regex(myTableFilterBox.Text);
foreach (IBlueRow r in rows) {
foreach (IBlueRow r in rows)
{
string thisTable = r["tag"].ToString().Substring(PubStreamPrefixer.prefix("").Length);
if (tableRegex.IsMatch(thisTable))
sl[thisTable] = true;
@@ -638,7 +719,8 @@ namespace BCG {
myTableListBox.EndUpdate();
}
private void testGridToolStripMenuItem_Click(object sender, EventArgs e) {
private void testGridToolStripMenuItem_Click(object sender, EventArgs e)
{
HighPerformanceTimer timer = new HighPerformanceTimer();
//while (true)
//{
@@ -660,9 +742,11 @@ namespace BCG {
foreach (IBlueRow r in s.rows)
rows.Add(r);
for (int i = 0; i < 100; i++) {
for (int i = 0; i < 100; i++)
{
timer.start();
rows.Sort(delegate (IBlueRow r1, IBlueRow r2) {
rows.Sort(delegate (IBlueRow r1, IBlueRow r2)
{
return _sort_dicts_index(r1, r2, 5, true);
});
timer.stop();
@@ -670,19 +754,22 @@ namespace BCG {
}
}
private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high) {
private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high)
{
if (r1.Contains(column) && r2.Contains(column))
return compare_send_dontcare_to_last(r1[column], r2[column], low_to_high);
else
return 0;
}
private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high) {
private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high)
{
//return compare_send_dontcare_to_last(r1[col_no], r2[col_no], low_to_high);
return ((IComparable)r1[col_no]).CompareTo(r2[col_no]);
}
internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) {
internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high)
{
if (myIsDontCare(o1))
return myIsDontCare(o2) ? 0 : 1;
else
@@ -690,14 +777,16 @@ namespace BCG {
: ((IComparable)o2).CompareTo(o1));
}
private static bool myIsDontCare(object o) {
private static bool myIsDontCare(object o)
{
return o == null || o == DBNull.Value ||
o is string && String.IsNullOrEmpty((string)o) ||
o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) ||
o is float && (Single.IsInfinity((float)o) || Single.IsNaN((float)o));
}
private void _new_grid_form(object args) {
private void _new_grid_form(object args)
{
//System.Timers.Timer t = new System.Timers.Timer();
////t.SynchronizingObject
//BCG.Utilities.BlueEventLog.logEvent(String.Format("New BTSTabularGridTestForm created on thread {0}", System.Threading.Thread.CurrentThread.ManagedThreadId),
@@ -709,11 +798,13 @@ namespace BCG {
private void libraryToolStripMenuItem_Click(object sender, EventArgs e) { }
private void myTableFilterLabel_Click(object sender, EventArgs e) {
private void myTableFilterLabel_Click(object sender, EventArgs e)
{
}
private void MainForm_Load(object sender, EventArgs e) {
private void MainForm_Load(object sender, EventArgs e)
{
}
}
@@ -5,16 +5,31 @@ using bts.utils;
using mbts.config;
using System;
using System.Linq;
using System.Windows.Forms;
using System.Threading.Tasks;
using System.ComponentModel;
static class Program {
[STAThread]
static void Main(string[] args) {
static void init(string[] args) {
Func<string, string, Task> login = async (l, p) => {
var login = l;
var passwd = p;
var service_ctx = bts.api.core.ServiceContext.create();
await service_ctx.login(login, passwd);
};
BTSNativeMethods.provide_command_line(args, args.Length);
Configuration.initialize();
bts.Log.Gen.info($"Starting DSQA");
Console.WriteLine("Starting DataStreamQueryAnalyzer...");
Console.WriteLine("Starting Test Publisher");
Task.Run(async () => await login("bts", String.Empty));
}
[STAThread]
static void Main(string[] args) {
init(args);
string args_concat = String.Empty;
args.ToList().ForEach(t => args_concat += t);
var init_log = $"Starting DSQA args: {args_concat} len: {args.Length}";
bts.Log.Gen.info(init_log);
Console.WriteLine(init_log);
Console.WriteLine("Launching Main Window");
MainForm mainForm = new MainForm();
if (args.Length > 0) {
@@ -9,8 +9,7 @@ using System.IO;
using System.Threading;
using mbts;
namespace BCG
{
namespace BCG {
/// <summary>
/// Keeps an ITCPConnection for talking to the server. It also monitors heartbeats
/// from the server, and in case of missing hbs, informs its clients, and tries to
@@ -23,8 +22,7 @@ namespace BCG
/// </summary>
public class PubSubServerManagedTCPConnection : IPubSubServerConnection,
ITCPMessageHandler,
ITCPConnectionHandler
{
ITCPConnectionHandler {
private ITCPConnection _connection, _new_connection;
private readonly BlutoSubAddress _server_address;
// if a message(with confirm) can not be sent , its id is queued here,
@@ -43,98 +41,75 @@ namespace BCG
public PubSubServerManagedTCPConnection(string serverURL)
: this(new BlutoSubAddress(serverURL)) { }
public PubSubServerManagedTCPConnection(BlutoSubAddress server_address)
{
public PubSubServerManagedTCPConnection(BlutoSubAddress server_address) {
_server_address = server_address;
_connect_to_ps_server();
}
private void _connect_to_ps_server()
{
private void _connect_to_ps_server() {
bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server: {_server_address}");
ClientToServerManagedTCPConnector.connect_to(_server_address, this, TimeSpan.FromSeconds(1));
}
#region IPubSubServerConnection Members
public void sendWithConfirmation(byte[] data, ref string conf_id)
{
public void sendWithConfirmation(byte[] data, ref string conf_id) {
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} myConnection : {_connection != null}");
if (_connection != null)
{
_connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) =>
{
if (_connection != null) {
_connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) => {
if (_on_send_onfirm_callback != null)
_on_send_onfirm_callback(msg_id, confirmed);
});
}
else
{
} else {
bts.Log.Gen.warn($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} no connection, buffering send");
conf_id = Guid.NewGuid().ToString();
_negative_unsent_confs.enqueue(conf_id);
}
}
public void send(byte[] data)
{
public void send(byte[] data) {
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_address} data: {data.Length} myConnection : {_connection != null}");
if (_connection != null)
{
if (_connection != null) {
bts.Log.Gen.info("PubSubServerManagedTCPConnection.send server: " + _server_address + "data: " + data.Length);
_connection.send(data);
}
}
public void send(Stream str)
{
public void send(Stream str) {
BinaryReader br = new BinaryReader(str);
send(br.ReadBytes(Convert.ToInt32(str.Length - str.Position)));
}
public void send(PubSubMessage msg)
{
public void send(PubSubMessage msg) {
send(msg.to_stream());
}
public void send_with_confirmation(PubSubMessage msg, ref string conf_id)
{
public void send_with_confirmation(PubSubMessage msg, ref string conf_id) {
var bytes = msg.to_bytes();
sendWithConfirmation(bytes, ref conf_id);
}
public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements)
{
public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements) {
return _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements);
}
public string serverURL
{
public string serverURL {
get { return _server_address.ToString(); }
}
public string clientURL
{
get
{
public string clientURL {
get {
string rv = "";
lock (this)
{
if (_connection == null)
{
lock (this) {
if (_connection == null) {
rv = "";
}
else
{
if (PubSub3Config.instance.PS3_IPTranslation == null || PubSub3Config.instance.PS3_IPTranslation == String.Empty)
{
} else {
if (PubSub3Config.instance.PS3_IPTranslation == null || PubSub3Config.instance.PS3_IPTranslation == String.Empty) {
string ip;
int port;
_connection.getSelfAddress(out ip, out port);
rv = new BlutoSubAddress(ip, port).ToString();
}
else
{
} else {
rv = "";
}
}
@@ -143,18 +118,15 @@ namespace BCG
}
}
public void registerCallback(ConnectionStatusChange del)
{
public void registerCallback(ConnectionStatusChange del) {
_conn_callback = del;
}
public void registerDataCallback(DataDelegate del)
{
public void registerDataCallback(DataDelegate del) {
_data_callback = del;
}
public void registerSendConfirmationCallback(SendConfirmationDelegate del)
{
public void registerSendConfirmationCallback(SendConfirmationDelegate del) {
_on_send_onfirm_callback = del;
}
@@ -163,21 +135,17 @@ namespace BCG
#region IDisposable Members
private bool _is_disposed = false;
public void Dispose()
{
public void Dispose() {
bts.Log.Gen.info("PubSubServerManagedTCPConnection.Dispose");
//throw new Exception("The method or operation is not implemented.");
if (_connection != null)
{
if (_connection != null) {
_connection.Dispose();
}
_is_disposed = true;
}
public bool isDisposed
{
get
{
public bool isDisposed {
get {
return _is_disposed;
}
@@ -193,12 +161,10 @@ namespace BCG
/// (long) is safe for multi threaded operations
/// </summary>
/// <param name="message"></param>
public void onData(RemoteMessage message)
{
public void onData(RemoteMessage message) {
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData message: {message}");
// TODO check for disconenction
switch (message.messageType)
{
switch (message.messageType) {
case RemoteMessageType.Data:
if (_data_callback != null)
_data_callback(message.remoteConnectionInfo.ToString(), message.message);
@@ -219,17 +185,18 @@ namespace BCG
break;
}
if (message.messageType == RemoteMessageType.Data)
{
if (_data_callback != null)
_data_callback(message.remoteConnectionInfo.ToString(), message.message);
else
_in_data_queue.enqueue(message.message);
}
else
{
_last_hb = DateTime.Now;
}
// this stupid shit, caused duplicating og messages and 2 days of wild goose chase
// if (message.messageType == RemoteMessageType.Data)
// {
// if (_data_callback != null)
// _data_callback(message.remoteConnectionInfo.ToString(), message.message);
// else
// _in_data_queue.enqueue(message.message);
// }
// else
// {
// _last_hb = DateTime.Now;
// }
}
#endregion
@@ -241,15 +208,13 @@ namespace BCG
/// except onData)
/// </summary>
/// <param name="connection"></param>
public void onConnection(ITCPConnection connection)
{
public void onConnection(ITCPConnection connection) {
string ip;
int port;
connection.getPeerAddress(out ip, out port);
BlutoSubAddress add = new BlutoSubAddress(ip, port);
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onConnection add: {add.ToString()} connection type: {connection.GetType()}");
if (add.ToString() == _server_address.ToString())
{
if (add.ToString() == _server_address.ToString()) {
_last_hb = DateTime.Now;
_new_connection = connection;
}
@@ -257,39 +222,31 @@ namespace BCG
#endregion
void _dispose_and_reconnect_if_missing_hb()
{
if (!_connection.isDisposed)
{
void _dispose_and_reconnect_if_missing_hb() {
if (!_connection.isDisposed) {
TimeSpan hb_interval = DateTime.Now - _last_hb;
bool hb_missed = hb_interval > _connection_drop_threshold;
if (hb_missed)
{
if (hb_missed) {
if (_conn_callback != null)
_conn_callback(_server_address.ToString(), false);
bts.Log.Gen.info("PubSubServerManagedTCPConnection._dispose_and_reconnect_if_missing_hb disposing connecion: {myServerAddress}");
_connection.Dispose();
_connect_to_ps_server();
}
}
else
{
} else {
// already disposed, nothing to do
}
}
void _check_for_new_connection()
{
if (_new_connection != null)
{
void _check_for_new_connection() {
if (_new_connection != null) {
_connection = _new_connection;
_new_connection = null; // since we are using its presence as a marker of nw connections
bts.Log.Gen.info("PubSubServerManagedTCPConnection._check_for_new_connection assigning new connection");
_connection.register(this);
if (_conn_callback != null)
_conn_callback(_server_address.ToString(), true);
if (_on_send_onfirm_callback != null)
{
if (_on_send_onfirm_callback != null) {
var msg_ids = _negative_unsent_confs.dequeueAll();
foreach (var id in msg_ids)
_on_send_onfirm_callback(id, false);
@@ -304,8 +261,7 @@ namespace BCG
/// the design only allows for onConnection to be on a different
/// thread.
/// </summary>
public void doWork()
{
public void doWork() {
//_dispose_and_reconnect_if_missing_hb();
_check_for_new_connection();
///check for heartbeats and let clients know in
@@ -304,6 +304,7 @@ namespace BCG {
}
private void myCollectMissingHeartbeatConnections() {
return; // disabling HBs for now to simplify things for the demo
if (DateTime.Now - myLastHeartbeatDeadCollectionTime > mySubscriberDropThreshold) {
myLastHeartbeatDeadCollectionTime = DateTime.Now;
///these are all the hbs received bw the last
@@ -321,10 +322,9 @@ namespace BCG {
if (!indexedHbs.ContainsKey(clientURL))
deadClients.Add(clientURL);
foreach (string deadClient in deadClients) {
string disposeMsg = String.Format("Collecting missing HB connections : {0} is dead due to missing hbs, Disposing it now, " +
string dispose_msg = String.Format("Collecting missing HB connections : {0} is dead due to missing hbs, Disposing it now, " +
"connection count before deleting this = {1}", deadClient, myConnections.Count);
//BlueEventLog.logEvent(disposeMsg, PS3LogLevels.disconnection);
Console.WriteLine(disposeMsg);
bts.Log.Gen.error(dispose_msg);
myConnections[deadClient].Dispose();
myConnections.Remove(deadClient);
myDeadConnectionURLs.enqueue(deadClient);
@@ -181,7 +181,7 @@ namespace BCG {
public void publish(string tagType, string tag, Stream content) {
PublisherCoreMessage msg = new PublisherCoreMessage(tagType, tag, content);
bts.Log.Gen.info($"StaticPublisherCore.publish msg: {msg}");
bts.Log.Gen.info($"StaticPublisherCore.publish tag: {tag} msg: {msg} len: {content.Length - content.Position}");
myMainQueue.enqueue(msg);
}
@@ -357,7 +357,7 @@ namespace BCG {
myState.publish(tag, tagType, content);
if (!myCrushableTagsWaitingOnAck.ContainsKey(tag)) {
bts.Log.Gen.info("Sending crushable tag: {tag}");
mySendData(tag, tagType, content);
mySendData(tag, tagType, content, "myPublish");
} else {
bts.Log.Gen.info("Could not send tag: {tag} since its waiting on an ack");
myCrushableTagsWaitingOnAck[tag].second++;
@@ -374,8 +374,8 @@ namespace BCG {
}
// add header, possibly compress the data, and sendto appropriate pubsub
private void mySendData(string tag, string tagType, Stream content) {
bts.Log.Gen.info($"StaticPublisherCore.mySendData tag: {tag} tagType: {tagType}");
private void mySendData(string tag, string tagType, Stream content, string reason) {
bts.Log.Gen.info($"StaticPublisherCore.mySendData tag: {tag} tagType: {tagType} reason: {reason} len: {content.Length - content.Position}");
long initPos = content.Position;
// locate right pubsub for this tag
IPubSubServerConnection serverConn = myGetOrBuildSender(tag);
@@ -407,7 +407,6 @@ namespace BCG {
serverConn.send_with_confirmation(ps_msg, ref refId);
myCrushableTagsWaitingOnAck.Add(tag, new Pair<string, int>(refId, 0));
} else {
//Console.WriteLine("StaticPublisherCore.mySend serverConn.send tag: " + tag + " server: " + serverConn.serverURL);
// this is where the data is actually sent out
var ps_msg = new PubSubMessage(pub_md, contentToPublish);
bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
@@ -440,6 +439,7 @@ namespace BCG {
///in case of the server coming up, it publishes all the stored state
///for that server, back to it
protected void myHandleServerReboot(string serverURL, bool connection) {
bts.Log.Gen.info($"StaticPublisherCore.myHandleServerReboot server_url: {serverURL}");
if (connection) {
string thisPublisherInfo = ClientInfo.thisClient.processString;
foreach (KeyValuePair<string, IPubSubServerConnection> p in myTagToSender) {
@@ -448,7 +448,7 @@ namespace BCG {
string tag = p.Key;
myCrushableTagsWaitingOnAck.Clear();
byte[] data = myState.getContent(tag, out tagType);
mySendData(tag, tagType, new MemoryStream(data));
mySendData(tag, tagType, new MemoryStream(data), "myHandleServerReboot");
//Stream content = new MemoryStream(data);
//Stream contentToPublish = content;
@@ -13,8 +13,7 @@ using mbts;
using mbts.spark;
//using BCG.Utilities.BlueConfiguration;
namespace BCG
{
namespace BCG {
/// <summary>
/// This class serves as the subscrition interface to the pubsub. The only methods
@@ -25,70 +24,56 @@ namespace BCG
/// to the servers even though there might be so created by other instances of the SubscriberCore
///
/// </summary>
public class StaticSubscriberCore : ISubscriberCore
{
public class StaticSubscriberCore : ISubscriberCore {
private Dictionary<string, ISingleServerSubscriberCore> myServerURLToSubCore = new Dictionary<string, ISingleServerSubscriberCore>();
private IPubSubRouter myRouter;
private HighPerformanceTimer myTimer = new HighPerformanceTimer();
private PubSubServerConnectionFactory myConnectionFactory = new PubSubServerConnectionFactory(true);
public event PubSubConnectionChange onPubSubConnectionChange;
#region constructors
public StaticSubscriberCore()
{
public StaticSubscriberCore() {
//this defaults to a basic pubsubrouter if the config key is not filled out
myInit(myGetRouter());
}
public StaticSubscriberCore(string siteName)
{
public StaticSubscriberCore(string siteName) {
myInit(new PubSubRouter(PubSub3Config.instance.getRouting(siteName)));
}
private IPubSubRouter myGetRouter()
{return new PubSubRouter(PubSub3Config.instance.getRouting());}
private IPubSubRouter myGetRouter() { return new PubSubRouter(PubSub3Config.instance.getRouting()); }
public StaticSubscriberCore(PubSubRoute[] routes)
{
public StaticSubscriberCore(PubSubRoute[] routes) {
myInit(new PubSubRouter(routes));
}
public StaticSubscriberCore(IPubSubRouter router)
{
public StaticSubscriberCore(IPubSubRouter router) {
myInit(router);
}
#endregion
private void myInit(IPubSubRouter router)
{
private void myInit(IPubSubRouter router) {
myRouter = router;
}
public ArrayList getUpdates()
{
public ArrayList getUpdates() {
return getUpdates(10);
}
public ArrayList getUpdates(TimeSpan timeout)
{
public ArrayList getUpdates(TimeSpan timeout) {
return getUpdates(Convert.ToInt32(timeout.TotalMilliseconds));
}
public ArrayList getUpdates(int timeout)
{
public ArrayList getUpdates(int timeout) {
//BCG.BlueTrace.WriteTraceMethodEnterW("StaticSubscriberCore.getUpdates");
lock (myServerURLToSubCore)
{
lock (myServerURLToSubCore) {
if (myServerURLToSubCore.Count == 0)
{
if (myServerURLToSubCore.Count == 0) {
return new ArrayList();
}
else
{
} else {
///gets a time share for every subscriber and
///calls their updates
myTimer.start();
@@ -96,6 +81,7 @@ namespace BCG
List<PubSubUpdate> retVal = new List<PubSubUpdate>();
foreach (ISingleServerSubscriberCore subCore in myServerURLToSubCore.Values)
retVal.AddRange(subCore.getUpdates(timeShare));
_log_n_updates(retVal);
myTimer.stop();
//if (myTimer.lastIntervalTime > 10)
// BlueEventLog.logEvent("get updates taking " + myTimer.lastIntervalTime, BlueEventLevel.Error);
@@ -104,30 +90,36 @@ namespace BCG
return new ArrayList(retVal);
}
}
}
private Dictionary<string,bool> myBlueCommTags = new Dictionary<string,bool> ();
bool myIsBlueCommTag(string tag)
{
return false;
private IDictionary<string, int> _n_update_map = new Dictionary<string, int>();
private void _log_n_updates(IEnumerable<PubSubUpdate> updates) {
foreach (PubSubUpdate u in updates) {
if (!_n_update_map.ContainsKey(u.tag)) {
_n_update_map.Add(u.tag, 0);
}
_n_update_map[u.tag] = _n_update_map[u.tag] + 1;
}
// log it now
foreach (var kv in _n_update_map) {
bts.Log.Gen.info($"StaticSubscrberCore tag: {kv.Key} n_updates: {kv.Value}");
}
}
public void subscribe(params string[] tags)
{
lock (myServerURLToSubCore)
{
foreach (string tag in tags)
{
if (myIsBlueCommTag(tag))
{
}
else
{
private Dictionary<string, bool> myBlueCommTags = new Dictionary<string, bool>();
bool myIsBlueCommTag(string tag) {
return false;
}
public void subscribe(params string[] tags) {
lock (myServerURLToSubCore) {
foreach (string tag in tags) {
if (myIsBlueCommTag(tag)) {
} else {
string serverURL = myRouter.getSubURLForTag(tag);
if (!myServerURLToSubCore.ContainsKey(serverURL))
{
if (!myServerURLToSubCore.ContainsKey(serverURL)) {
// BlueEventLog.logEvent("Creating a SingleServerSubscriberCore for " + serverURL, BlueEventLevel.Information);
myServerURLToSubCore.Add(serverURL, new SingleServerSubscriberCore(serverURL, myConnectionFactory));
myServerURLToSubCore[serverURL].onPubSubConnectionChange += myHandleConnectionChange;
@@ -138,28 +130,22 @@ namespace BCG
}
}
}
public void myHandleConnectionChange (bool isConnected, string connection)
{
public void myHandleConnectionChange(bool isConnected, string connection) {
if (onPubSubConnectionChange != null)
onPubSubConnectionChange(isConnected, connection);
}
public void unsubscribe(params string[] tags)
{
lock (myServerURLToSubCore)
{
public void unsubscribe(params string[] tags) {
lock (myServerURLToSubCore) {
foreach (string tag in tags)
{
foreach (string tag in tags) {
if (myIsBlueCommTag(tag))
myServerURLToSubCore["bluecomm"].unsubscribe(tag);
else
{
else {
string serverURL = myRouter.getSubURLForTag(tag);
if (!myServerURLToSubCore.ContainsKey(serverURL))
{
if (!myServerURLToSubCore.ContainsKey(serverURL)) {
// BlueEventLog.logEvent("Creating a SingleServerSubscriberCore for " + serverURL, BlueEventLevel.Information);
myServerURLToSubCore.Add(serverURL, new SingleServerSubscriberCore(serverURL, myConnectionFactory));
}
@@ -168,15 +154,13 @@ namespace BCG
}
}
}
public IPubSubRouter getRouter()
{
public IPubSubRouter getRouter() {
return myRouter;
}
#region IDisposable Members
public void Dispose()
{
public void Dispose() {
foreach (ISingleServerSubscriberCore sssc in myServerURLToSubCore.Values)
sssc.Dispose();
@@ -40,7 +40,7 @@ namespace BCG {
IReadOnlyDictionary<SecurityKey, BookTuple> _cached_state = null;
MarketDataClient _md_client;
HighPerformanceTimer _timer = new HighPerformanceTimer();
private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo",
private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("books",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("sec_key", TypeCode.UInt32),
Tuple.Create("level", TypeCode.UInt32),
@@ -1,20 +1,26 @@
namespace BCG {
namespace BCG
{
using bts.api.orderman;
enum TSGroupByOperation {
enum TSGroupByOperation
{
OHLC = 0,
Statistics
}
[DataStreamUserRespresentation("timeseriesanalysis")]
public class TimeSeriesGroupByStream : ViewStream {
public TimeSeriesGroupByStream(ViewStream md_str, string timestamp_col, string output_col, string operation) {
ViewStream rs = new NonIncGroupByStream(md_str, new OHLCNonIncGroupByEvaluator(timestamp_col, output_col));
public class TimeSeriesGroupByStream : ViewStream
{
public TimeSeriesGroupByStream(ViewStream md_str, string timebucket_col, string output_col, string operation)
{
ViewStream rs = new NonIncGroupByStream(md_str, new OHLCNonIncGroupByEvaluator(timebucket_col, output_col));
mySetRealSource(rs);
}
}
public class OHLC {
public OHLC(double open_args, double high_args, double low_args, double close_args) {
public class OHLC
{
public OHLC(double open_args, double high_args, double low_args, double close_args)
{
open = open_args;
high = high_args;
low = low_args;
@@ -22,36 +28,47 @@ namespace BCG {
}
public readonly double open, high, low, close;
}
public class OHLCNonIncGroupByEvaluator : INonIncGroupByEvaluator {
public class OHLCNonIncGroupByEvaluator : INonIncGroupByEvaluator
{
private string _timestamp_col;
private string _output_col;
private IBlueTableSchema _out_schema;
public OHLCNonIncGroupByEvaluator(string timestamp_col, string output_col) {
public OHLCNonIncGroupByEvaluator(string timestamp_col, string output_col)
{
_timestamp_col = timestamp_col;
_output_col = output_col;
}
string open_col {
get {
string open_col
{
get
{
return $"{_output_col}_open"; ;
}
}
string high_col {
get {
string high_col
{
get
{
return $"{_output_col}_high"; ;
}
}
string low_col {
get {
string low_col
{
get
{
return $"{_output_col}_low"; ;
}
}
string close_col {
get {
string close_col
{
get
{
return $"{_output_col}_close"; ;
}
}
private void _build_schema(IBlueTableSchema input_schema) {
private void _build_schema(IBlueTableSchema input_schema)
{
List<Tuple<Type, string>> new_cols = new List<Tuple<Type, string>>();
ColumnInfo ci = input_schema[_output_col];
new_cols.Add(Tuple.Create(ci.colType, open_col));
@@ -62,23 +79,30 @@ namespace BCG {
_out_schema = BlueDataUtility.add_columns(input_schema, new_cols.ToArray());
}
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
if (_out_schema == null) {
if (grouped_rows.Count > 0) {
foreach (var kv in grouped_rows) {
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows)
{
if (_out_schema == null)
{
if (grouped_rows.Count > 0)
{
foreach (var kv in grouped_rows)
{
var rows = kv.Value;
if (rows != null && rows.Count > 0) {
if (rows != null && rows.Count > 0)
{
_build_schema(rows.ElementAt(0).schema);
}
}
}
}
BlueTable rv = new BlueTable(_out_schema);
foreach (var kv in grouped_rows) {
foreach (var kv in grouped_rows)
{
int id = kv.Key;
IEnumerable<IBlueRow> rows = kv.Value;
// sort rows on time to compute o,c correctly??
if (rows != null && rows.Count() > 0) {
if (rows != null && rows.Count() > 0)
{
IBlueRow reprow = rows.ElementAt(0);
IEnumerable<double> vals = BlueDataUtility.get_covalues<double>(rows, _output_col);
OHLC ohlc = compute_ohlc(vals);
@@ -94,11 +118,13 @@ namespace BCG {
return rv;
}
public string[] get_group_by_cols() {
public string[] get_group_by_cols()
{
return new string[] { _timestamp_col };
}
static OHLC compute_ohlc(IEnumerable<double> prices) {
static OHLC compute_ohlc(IEnumerable<double> prices)
{
double o = prices.ElementAt(0);
double c = prices.ElementAt(prices.Count() - 1);
double l = prices.Min();
@@ -106,11 +132,30 @@ namespace BCG {
return new(o, h, l, c);
}
}
[DataStreamUserRespresentation("ohlc")]
public class OHLCStream : ViewStream {
[DataStreamUserRespresentation("RSI")]
public class RSIStream : ViewStream
{
// for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
public OHLCStream(ViewStream md_str, string timestamp_col, string output_col) {
ViewStream rs = new TimeSeriesGroupByStream(md_str, timestamp_col, output_col, TSGroupByOperation.OHLC.ToString());
public RSIStream(ViewStream md_str, string timebucket_column, string output_col)
{
}
}
[DataStreamUserRespresentation("Bollinger")]
public class BollingerBandStream : ViewStream
{
// for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
public BollingerBandStream(ViewStream md_str, string timebucket_column, string output_col)
{
}
}
[DataStreamUserRespresentation("OHLC")]
public class OHLCStream : ViewStream
{
// for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
public OHLCStream(ViewStream md_str, string timebucket_column, string output_col)
{
ViewStream rs = new TimeSeriesGroupByStream(md_str, timebucket_column, output_col, TSGroupByOperation.OHLC.ToString());
mySetRealSource(rs);
}
}
@@ -1,4 +1,5 @@
namespace BCG {
namespace BCG
{
using BCG.Utilities.General;
using bts.api;
@@ -19,7 +20,8 @@ namespace BCG {
using SecurityKey = System.UInt32;
[DataStreamUserRespresentation("timesales")]
public class TimeSalesStream : ViewStream, IWorker {
public class TimeSalesStream : ViewStream, IWorker
{
private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
private DateTime _last_poll_time = DateTime.MinValue;
private ViewStreamDatabase _vsd;
@@ -28,37 +30,43 @@ namespace BCG {
MarketDataClient _md_client;
private bool _dirty = false;
private ConcurrentQueue<TradeInfo> _cached_trades = new ConcurrentQueue<TradeInfo>();
private BlueTable _table = new BlueTable(BlueDataUtility.form_schema("instruments",
private readonly int _buffer_size;
private IBlueTableSchema _schema = BlueDataUtility.form_schema("TimeSales",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("sec_key", TypeCode.UInt32),
Tuple.Create("sec", TypeCode.String),
Tuple.Create("time", TypeCode.Int64),
Tuple.Create("time", TypeCode.DateTime),
Tuple.Create("price", TypeCode.Double),
Tuple.Create("qty", TypeCode.UInt32),
Tuple.Create("aggressor", TypeCode.String)
));
public TimeSalesStream()
: this(ViewStreamDatabase.instance) {
Tuple.Create("aggressor", TypeCode.String));
public TimeSalesStream(int buffer_size)
: this(buffer_size, ViewStreamDatabase.instance)
{
}
public TimeSalesStream(ViewStreamDatabase vsd) {
public TimeSalesStream(int buffer_size, ViewStreamDatabase vsd)
{
_buffer_size = buffer_size;
bts.Log.Gen.info("TimeSalesStream.cons");
_vsd = vsd;
_md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
}
protected override void myOnFirstSubscriber() {
protected override void myOnFirstSubscriber()
{
_vsd.addWorker(this);
_md_client.trades(new TradesSubscriptionFilter()).Subscribe(_on_trades);
//_md_client.trades(null).Subscribe(_on_trades);
}
private void _log_updates(IList<TradeInfo> trades) {
private void _log_updates(IList<TradeInfo> trades)
{
bts.Log.Gen.info($"TimeSalesStream update sz: {trades.Count}");
}
private void _on_trades(IList<TradeInfo> trades) {
private void _on_trades(IList<TradeInfo> trades)
{
_dirty = true;
_log_updates(trades);
// just cacching in here, will populate the rows in _poll_scan and fire the datapacket
@@ -67,23 +75,35 @@ namespace BCG {
_cached_trades.Enqueue(t);
}
protected override void myOnLastSubscriber() {
protected override void myOnLastSubscriber()
{
_vsd.removeWorker(this);
}
private void _scan_trades() {
BlueTable added_rows = new BlueTable(_table.schema);
foreach (TradeInfo tinf in _cached_trades) {
IBlueRow r = new BlueRow(_table.schema, _id++, tinf.security.Id, tinf.security.ToString(), tinf.trade.Time, tinf.trade.Mantissa, tinf.trade.Size, tinf.trade.AggressorSide.ToString());
added_rows.addRow(r);
private void _scan_trades()
{
Func<TradeInfo, DateTime> to_dt = (tinf) =>
{
return new mbts.utils.LLDateTime((ulong)tinf.trade.Time).to_datetime().ToLocalTime();
};
List<TradeInfo> trades = new List<TradeInfo>(_cached_trades);
var latest_n_trades = trades.OrderByDescending(t => (ulong)t.trade.Time).Take(_buffer_size).ToList();
BlueTable new_state = new BlueTable(_schema);
foreach (TradeInfo tinf in Enumerable.Reverse(latest_n_trades))
{
// TODO, time
IBlueRow r = new BlueRow(_schema, _id++, tinf.security.Id, tinf.security.ToString(), to_dt(tinf), tinf.trade.Mantissa * Math.Pow(10, tinf.trade.Exponent), tinf.trade.Size, tinf.trade.AggressorSide.ToString());
new_state.addRow(r);
}
_table.addRows(added_rows);
if (added_rows.Count > 0)
myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
DataPacket dp = BlueDataUtility.build_dp_inc(new_state, myLastValue as DataPacket);
myFireEvent(dp);
}
public void doWork() {
if (_dirty && DateTime.Now - _last_poll_time > _poll_period) {
public void doWork()
{
if (_dirty && DateTime.Now - _last_poll_time > _poll_period)
{
_last_poll_time = DateTime.Now;
_dirty = false;
_scan_trades();
@@ -45,7 +45,7 @@ namespace BCG
#region constructors to be used by the static factory method
private PubSubServerTagStatsStream(string serverURL)
public PubSubServerTagStatsStream(string serverURL)
: this(ViewStreamDatabase.instance, serverURL)
{ }