From 88bf04e5f1930281a78eefec2091f92f00ea9ae5 Mon Sep 17 00:00:00 2001 From: Taha Date: Mon, 23 Oct 2023 14:18:31 -0400 Subject: [PATCH] TimeSeries and TimeBucketStreams, Stream offloading etc --- .../DataStream/Offloading/AddToCloudStream.cs | 24 + .../Offloading/CloudStreamManager.cs | 100 ++ .../Offloading/StreamOffloadRequestor.cs | 59 ++ .../Join/NonIncremental/NonIncJoinStream.cs | 9 +- .../DataStream/Primitives/Misc/DebugStream.cs | 2 + .../Primitives/Misc/TimeSeriesStream.cs | 142 ++- .../SideEffectStreams/CSVSaverStream.cs | 2 +- .../DataStream/Primitives/Time/ThrottleStream | 0 .../Primitives/Time/TimeBucketStream | 51 + .../Primitives/Time/TimeUnitStream.cs | 78 +- .../DataStream/Publish/PubTableBroker.cs | 37 +- .../DataStream/Publish/ViewPublisher.cs | 2 +- .../BlueData/BlueDataUtility.cs | 889 ++++++++++++------ .../DataStreamCore/BlueData/BlueTable.cs | 84 +- .../Accumulation/TableAccumulator.cs | 3 +- .../ClientDeserializationController.cs | 64 +- .../ClientUniversalRowDeserializer.cs | 64 +- .../ClientReceive/IClientDeserializer.cs | 141 ++- .../ClientReceive/IClientRowDeserializer.cs | 8 +- .../Messaging/Publish/PubTablePublisher.cs | 11 +- .../DataStreamPublisher/Program.cs | 98 +- .../DataStreamQueryAnalyzer/MainForm.cs | 305 +++--- .../DataStreamQueryAnalyzer/Program.cs | 25 +- .../PubSubServerManagedTCPConnection.cs | 150 ++- .../ManagedTCP/ManagedTCPMessageHub.cs | 6 +- .../PubSub/StaticPublisherCore.cs | 12 +- .../PubSub/StaticSubscriberCore.cs | 116 +-- .../Business/MarketData/BookStream.cs | 2 +- .../TimeSeriesGroupByStream.cs | 107 ++- .../Business/MarketData/TimeSalesStream.cs | 66 +- .../PS3/Stats/PubSubServerTagStatsStream.cs | 2 +- 31 files changed, 1823 insertions(+), 836 deletions(-) create mode 100644 src/DataStreamSuite/DataStream/Offloading/AddToCloudStream.cs create mode 100644 src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs create mode 100644 src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs create mode 100644 src/DataStreamSuite/DataStream/Primitives/Time/ThrottleStream create mode 100644 src/DataStreamSuite/DataStream/Primitives/Time/TimeBucketStream diff --git a/src/DataStreamSuite/DataStream/Offloading/AddToCloudStream.cs b/src/DataStreamSuite/DataStream/Offloading/AddToCloudStream.cs new file mode 100644 index 000000000..87c8f9f52 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Offloading/AddToCloudStream.cs @@ -0,0 +1,24 @@ +using BCG; + +namespace BCG +{ + // this is a stream so we can run it from the DSQA (since dsqa can only load streams) + [DataStreamUserRespresentation("add_to_cloud")] + public class AddToCloudStream : ViewStream + { + private readonly string _name, _stream_exp; + private StreamOffloadRequestor _requestor = new StreamOffloadRequestor(); + public AddToCloudStream(string name, string stream_exp) + : this(name, stream_exp, 0) + { + } + + public AddToCloudStream(string name, string stream_exp, int persist) + { + _name = name; + _stream_exp = stream_exp; + _requestor.add_to_cloud(_name, _stream_exp, persist == 1); + mySetRealSource(new TableStream("cloud_streams")); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs b/src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs new file mode 100644 index 000000000..7c2b536cd --- /dev/null +++ b/src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs @@ -0,0 +1,100 @@ +using BCG; + +using System; +using System.Collections.Generic; + +using mbts.config; +using bts.utils; +using System.Threading.Tasks; +using Bts.DataStreamExpressions; +using System.Collections.Generic; +using BCG.Utilities.General; + +namespace BCG +{ + // listens to the requests posted on "cloud_streams" request table, and + // if unfulfilled starts a ViewPubsliher + public class CloudStreamManager : IWorker + { + public class StreamPubUnit + { + public string name; + public string stream_expression; + public ViewStream stream; + public bool persist; + public ViewPublisher view_publisher; + + public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) + { + name = name_; + stream_expression = stream_expression_; + stream = stream_; + persist = persist_; + view_publisher = view_publisher_; + } + } + static ViewStream build_stream(string stream_expression) + { + try + { + var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); + var rv = exp.getView(new Dictionary(0)); + return rv; + + } + catch (Exception ex) + { + bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}"); + return null; + } + } + private IDictionary + _stream_pub_map = new Dictionary(); + private ViewStream _stream = new TableStream("cloud_streams"); + private PubTableBroker _broker = new PubTableBroker(); + + public CloudStreamManager() + { + _stream.onChange += _on_str; + + } + private void _on_str(object obj) + { + DataPacket dp = obj as DataPacket; + foreach (IBlueRow r in dp.newRows) + { + string name = r.get("name"); + bool persist = r.get("persist"); + string stream_exp = r.get("stream_exp"); + if (!_stream_pub_map.ContainsKey(name)) + { + _assign(name, stream_exp, persist); + } + } + } + + private void _assign(string name, string stream_expression, bool persist) + { + bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}"); + ViewStream stream = build_stream(stream_expression); + _expose_to_ws(name, stream); + if (persist) + { + stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs + } + ViewPublisher view_pub = new ViewPublisher(name, _broker, stream); + StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub); + _stream_pub_map.Add(name, u); + } + + private void _expose_to_ws(string name, ViewStream stream) + { + // TODO: john + } + + public void doWork() + { + + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs b/src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs new file mode 100644 index 000000000..bf4c10119 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs @@ -0,0 +1,59 @@ +using BCG; + +using System.Linq; + +namespace BCG +{ + public class StreamOffloadRequestor + { + public class OffloadOp + { + string name; + string stream_exp; + } + private IBlueTableSchema _schema; + private IBlueRowCollection _rows; + private PubTableBroker _broker = new PubTableBroker(); + ViewStream _existing_offloads_vs = new TableStream("cloud_streams"); + public StreamOffloadRequestor() + { + _schema = new BlueTableSchema("cloud_streams", + new ColumnInfo("id", 0, typeof(int)), + new ColumnInfo("name", 1, typeof(string)), + new ColumnInfo("persist", 2, typeof(bool)), + new ColumnInfo("stream_exp", 3, typeof(string))); + _existing_offloads_vs.onChange += _on_str; + } + private void _on_str(object obj) + { + DataPacket dp = obj as DataPacket; + _rows = dp.newRows; + // this is needed so the broker knows the state of the table + _broker.createTable(_rows); + } + public void add_to_cloud(string name, string stream_exp, bool persist) + { + // check if it exists + if (_rows == null) + { + BlueTable table = new BlueTable(_schema); + IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp); + table.addRow(nr); + _broker.createTable(table); + // create table + } + else + { + if (exists(name)) + return; + IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp); + _broker.addRow(nr); + } + } + + bool exists(string name) + { + return _rows.to_ienum().Where(r => name == r.get("name")).Count() > 0; + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs index 1851832c2..0ba3113a0 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs @@ -8,6 +8,7 @@ using mbts.spark; namespace BCG { + [DataStreamUserRespresentation("fastJoin")] public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker { private readonly NonIncJoinSpec[] _join_specs; @@ -23,7 +24,7 @@ namespace BCG public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj, ViewStream right, string right_join_cols, string right_proj) - : this(new Triplet[] + : this(new Triplet[] { new Triplet (left, left_join_cols, left_proj), new Triplet (right, right_join_cols, right_proj) @@ -31,7 +32,7 @@ namespace BCG { } public NonIncJoinStream(params Triplet[] join_specs) - : this (NonIncJoinSpec.from_triplet (join_specs), null, 0) + : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { } public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms, @@ -102,8 +103,8 @@ namespace BCG { List cis = new List(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }); foreach (NonIncJoinBookkeeper d in _book_keeping_data) - d.constribute_cols_to_schema (cis); - _output_schema = new BlueTableSchema ("NonIncJoinStream", cis.ToArray()); + d.constribute_cols_to_schema(cis); + _output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray()); } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/DebugStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/DebugStream.cs index 3543fac6a..5fd4a25f6 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/DebugStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/DebugStream.cs @@ -24,6 +24,8 @@ namespace BCG { private void _onstream_change(object o) { DataPacket dp = o as DataPacket; + BlueDataUtility.validate_table_schema(dp.newRows); + BlueDataUtility.validate_table_schema(dp.addedRows); if (_last_recv_rows == null) { _last_recv_rows = dp.newRows; } else { diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs index 5ea4537ff..88ba32d32 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs @@ -1,37 +1,47 @@ using Google.Protobuf.WellKnownTypes; using System; +using BCG.Utilities.General; -namespace BCG { +namespace BCG +{ // Keeps adding changing rows to the stream, hence preserving the history // and creating a time series, for example this can be used to make bbo updates // into a time series of prices/qty - public class TimeSeriesStream : ViewStream { + public class TimeSeriesStream : ViewStream + { private BlueTable _table; private ViewStream _stream; private int _id = 0; - public TimeSeriesStream(ViewStream stream) { + public TimeSeriesStream(ViewStream stream) + { _stream = stream; } - protected override void myOnFirstSubscriber() { + protected override void myOnFirstSubscriber() + { _stream.onChange += new ChangeDelegate(_onstream_change); } - protected override void myOnLastSubscriber() { + protected override void myOnLastSubscriber() + { _stream.onChange -= new ChangeDelegate(_onstream_change); } - private void _onstream_change(object o) { + + private void _onstream_change(object o) + { DataPacket dp = o as DataPacket; DateTime now = DateTime.Now; - if (_table == null) { + if (_table == null) + { IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName); schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp")); _table = new BlueTable(schema); } BlueTable added_rows = new BlueTable(_table.schema); - foreach (IBlueRow r in dp.addedRows) { + foreach (IBlueRow r in dp.addedRows) + { MutableBlueRow ar = new MutableBlueRow(_table.schema, r); ar.setValue("timestamp", now); ar.setValue("id", _id++); @@ -42,4 +52,118 @@ namespace BCG { } } -} \ No newline at end of file + + public class TimeBucketStream : ViewStream, IStreamMapper + { + private ViewStream _src; + private readonly string _time_stamp_col; + private IBlueTableSchema _schema; + private int _bucket_size_seconds; + + public TimeBucketStream(ViewStream input, string time_stamp_col, int bucket_size_seconds) + { + _time_stamp_col = time_stamp_col; + _bucket_size_seconds = bucket_size_seconds; + var vs = new MapStream(input, this); + mySetRealSource(vs); + } + + #region IStreamMapper Members + public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) + { + IBlueTableSchema rv = input_schema; + //rv = BlueDataUtility.remove_columns(rv, _time_stamp_col); + rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int)); + return rv; + + } + + #endregion + + #region IDynamicStreamMapper Members + + public IBlueRow map(IBlueRow row) + { + DateTime ts = row.get(_time_stamp_col); + TimeSpan time_of_day = (ts - ts.Date); + int seconds = Convert.ToInt32(time_of_day.TotalSeconds); + double buckets = seconds / _bucket_size_seconds; + int bucket_number = Convert.ToInt32(Math.Floor(buckets)); + MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row); + rv.setValue("bucket_number", bucket_number); + return rv; + } + + #endregion + } + + // public class ThrottleStream : ViewStream, IWorker + // { + // private ViewStreamDatabase _vsd; + // private ViewStream _stream; + // private int _throttle_ms; + // private TimeSpan _min_fire_period; + // private DateTime _last_Fire_time = DateTime.MinValue; + // private DataPacket _unfired_dp; + // public ThrottleStream(ViewStreamDatabase vsd, ViewStream base_stream, int throttle_ms) + // { + // _vsd = vsd; + // _stream = base_stream; + // _throttle_ms = throttle_ms; + // _min_fire_period = TimeSpan.FromMilliSeconds(throttle_ms); + // } + + // protected override void myOnFirstSubscriber() + // { + // _vsd.addWorker(this); + // _stream.onChange += new ChangeDelegate(_onstream_change); + // } + // protected override void myOnLastSubscriber() + // { + // _vsd.removeWorker(this); + // _stream.onChange -= new ChangeDelegate(_onstream_change); + // } + + // private void _onstream_change(object obj) + // { + // DataPacket dp = obj as DataPacket; + // if (DateTime.Now - _last_Fire_time >= _min_fire_period) + // { + // // ok to fire here + // } + // else + // { + // _last_cached_state = dp.newRows; + // } + // _possibly_fire(dp.newRows); + // } + + // public void doWork() + // { + // if (_unfired_dp != null) + // { + + // } + // } + + // private void _possibly_fire() + // { + // if (_unfired_dp != null) + // { + // if (DateTime.Now - _last_fire_time > _min_fire_period) + // { + // DataPacket dp = BlueDataUtility.build_dp_inc(_unfired_dp.newRows, myLastValue); + // _unfired_dp = null; + // _last_Fire_time = DateTime.Now; + // myFireEvent(dp); + // } + // } + // } + + // } + + +} + + + diff --git a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs index cfe814f39..5991a25e0 100644 --- a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs @@ -6,7 +6,7 @@ using BCG.Utilities; namespace BCG { - [DataStreamUserRespresentation("csvSaver")] + [DataStreamUserRespresentation("CSVSaver")] public class CSVSaverStream : ViewStream, ISideEffectStream { private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)), diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/ThrottleStream b/src/DataStreamSuite/DataStream/Primitives/Time/ThrottleStream new file mode 100644 index 000000000..e69de29bb diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/TimeBucketStream b/src/DataStreamSuite/DataStream/Primitives/Time/TimeBucketStream new file mode 100644 index 000000000..be054548f --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Time/TimeBucketStream @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Collections; + +namespace BCG +{ + [DataStreamUserRepresentation("timeBucket")] + public class TimeBucketStream : ViewStream, IStreamMapper + { + private ViewStream _src; + private readonly string _time_stamp_col; + private IBlueTableSchema _schema; + private uint _bucket_size_seconds; + + public TimeBucketStream(ViewStream input, string time_stamp_col, uint bucket_size_seconds) + { + _time_stamp_col = time_stamp_col; + _bucket_size_seconds = bucket_size_seconds; + } + + #region IStreamMapper Members + public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) + { + IBlueTableSchema rv = input_schema; + //rv = BlueDataUtility.remove_columns(rv, _time_stamp_col); + rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int)); + return rv; + + } + + #endregion + + #region IDynamicStreamMapper Members + + public IBlueRow map(IBlueRow row) + { + DateTime ts = row.get(_time_stamp_col); + TimeSpan time_of_day = (ts - ts.Date) + int seconds = Convert.ToInt32(time_of_day.TotalSeconds); + int buckets = seconds / _bucket_size_seconds; + int bucket_number = Math.Floor(buckets); + MutableBlueRow rv = new MutableBlueRow(row, getOutputSchema(row.schema)) + rv.setValue("bucket_number", bucket_number); + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/TimeUnitStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/TimeUnitStream.cs index 25d37df0d..e37c475bf 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Time/TimeUnitStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Time/TimeUnitStream.cs @@ -1,10 +1,9 @@ using System; + using BCG.Utilities.General; -namespace BCG -{ - public abstract class TimeUnitStream : ViewStream, IWorker - { +namespace BCG { + public abstract class TimeUnitStream : ViewStream, IWorker { protected TimeSpan myOffset; protected DateTime myConstructionTime; protected int mySizeOfStream; @@ -13,87 +12,80 @@ namespace BCG protected ViewStreamDatabase myDb; protected int myId = 0; - public TimeUnitStream (DateTime zeroTime, TimeSpan offSet, int sizeOfStream, ViewStreamDatabase db) - { + public TimeUnitStream(DateTime zeroTime, TimeSpan offSet, int sizeOfStream, ViewStreamDatabase db) { myConstructionTime = DateTime.Now; myZeroTime = zeroTime; myOffset = offSet; mySizeOfStream = sizeOfStream; myDb = db; - myCollection = new CircularRowCollection ( - new BlueTableSchema ("Time", - new ColumnInfo ("id", 0, typeof(Int32)), new ColumnInfo ("time", 1, typeof(DateTime))), + myCollection = new CircularRowCollection( + new BlueTableSchema("Time", + new ColumnInfo("id", 0, typeof(Int32)), new ColumnInfo("time", 1, typeof(DateTime))), mySizeOfStream); - } + } #region ViewStream overriden methods - protected override void myOnFirstSubscriber() - { - myDb.addWorker (this); - myInit (); + protected override void myOnFirstSubscriber() { + myDb.addWorker(this); + myInit(); } - protected override void myOnLastSubscriber() - { - myDb.removeWorker (this); + protected override void myOnLastSubscriber() { + myDb.removeWorker(this); } #endregion - protected virtual void myInit () - { + protected virtual void myInit() { int periods = (int) - (DateTime.Now - myConstructionTime).TotalMilliseconds / (int) myOffset.TotalMilliseconds; - + (DateTime.Now - myConstructionTime).TotalMilliseconds / (int)myOffset.TotalMilliseconds; - DateTime dt = myGetLatestRow() == null ? myRoundTime(myZeroTime) : - myRoundTime (myZeroTime + new TimeSpan(myOffset.Ticks * periods)); + + DateTime dt = myGetLatestRow() == null ? myRoundTime(myZeroTime) : + myRoundTime(myZeroTime + new TimeSpan(myOffset.Ticks * periods)); if (myGetLatestRow() == null || - !myGetLatestRow()["time"].Equals(dt)) - { - IBlueRow ar = new BlueRow (myCollection.schema, + !myGetLatestRow()["time"].Equals(dt)) { + IBlueRow ar = new BlueRow(myCollection.schema, myId++, dt); - IBlueRow dr = myCollection.addRowWithReturn (ar); - myFireEvent (new DataPacket (myCollection, - myConvertToCollection (ar), - myConvertToCollection (dr))); + IBlueRow dr = myCollection.addRowWithReturn(ar); + if (n_updates <= 3000) + myFireEvent(new DataPacket(myCollection, + myConvertToCollection(ar), + myConvertToCollection(dr))); } } #region IWorker methods - public void doWork () - { - myInit (); + public void doWork() { + myInit(); } #endregion - protected abstract DateTime myRoundTime (DateTime dt); - + protected abstract DateTime myRoundTime(DateTime dt); + #region Private Helper methods - private IBlueRow myGetLatestRow () - { + private IBlueRow myGetLatestRow() { return myCollection.Count == 0 ? null : - BlueDataUtility.extractRow (myCollection, myCollection.Count - 1); + BlueDataUtility.extractRow(myCollection, myCollection.Count - 1); } - private IBlueRowCollection myConvertToCollection (IBlueRow r) - { - BlueTable retVal = new BlueTable (myCollection.schema); + private IBlueRowCollection myConvertToCollection(IBlueRow r) { + BlueTable retVal = new BlueTable(myCollection.schema); if (r != null) - retVal.addRow (r); + retVal.addRow(r); return retVal; } #endregion - } + } } diff --git a/src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs b/src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs index d6b28d187..a95ccc2a6 100644 --- a/src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs +++ b/src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs @@ -1,6 +1,8 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Linq.Expressions; +using System.Xml.Serialization; using BCG.Utilities; @@ -53,7 +55,12 @@ namespace BCG { } public virtual void createTable(IBlueRowCollection newTable) { - bts.Log.Gen.info($"PubTableBroker.createTable table: {newTable.schema.tableName}"); + string log_msg = $"PubTableBroker.createTable table: {newTable.schema.tableName}\n"; + foreach (IBlueRow r in newTable) { + log_msg += $"id: {r["id"]}"; + } + bts.Log.Gen.info(log_msg); + _check_schema_consistency(newTable.schema); myCheckTransaction(); IPubTablePublisher table_publisher = null; var name_key = newTable.schema.tableName.ToLower(); @@ -71,6 +78,9 @@ namespace BCG { /// /// public virtual void addRow(IBlueRow r) { + string log_msg = $"PubTableBroker.addRow table: {r.schema.tableName} id: {r["id"]}"; + bts.Log.Gen.info(log_msg); + _check_schema_consistency(r.schema); myCheckTransaction(); IPubTablePublisher table_publisher = null; var name_key = r.schema.tableName.ToLower(); @@ -86,6 +96,9 @@ namespace BCG { /// /// public virtual void deleteRow(IBlueRow r) { + string log_msg = $"PubTableBroker.deleteRow table: {r.schema.tableName} id: {r["id"]}"; + bts.Log.Gen.info(log_msg); + _check_schema_consistency(r.schema); myCheckTransaction(); IPubTablePublisher table_publisher = null; var name_key = r.schema.tableName.ToLower(); @@ -101,6 +114,9 @@ namespace BCG { /// /// public virtual void changeRow(IBlueRow r) { + string log_msg = $"PubTableBroker.changeRow table: {r.schema.tableName} id: {r["id"]}"; + bts.Log.Gen.info(log_msg); + _check_schema_consistency(r.schema); myCheckTransaction(); IPubTablePublisher table_publisher = null; var name_key = r.schema.tableName.ToLower(); @@ -118,7 +134,16 @@ namespace BCG { public virtual void makeBatchChanges(IBlueRowCollection newRows, IBlueRowCollection deletedRows, IBlueRowCollection addedRows) { - myCheckTransaction(); + string log_msg = $"PubTableBroker.makeBatchChanges {deletedRows.Count} {addedRows.Count} table: {newRows.schema.tableName} deleted_rows: "; + foreach (IBlueRow r in deletedRows) + log_msg += $"id: {r["id"]} "; + log_msg += "addedRows: "; + foreach (IBlueRow r in addedRows) + log_msg += $"id: {r["id"]}"; + bts.Log.Gen.info(log_msg); + _check_schema_consistency(newRows.schema); + _check_schema_consistency(deletedRows.schema); + _check_schema_consistency(addedRows.schema); IPubTablePublisher table_publisher = null; var name_key = newRows.schema.tableName.ToLower(); if (!_publisher_map.TryGetValue(name_key, out table_publisher)) @@ -144,8 +169,16 @@ namespace BCG { } protected virtual IPubTablePublisher _create_pubtable_publisher(string tag) { + bts.Log.Gen.info($"PubTableBroker._create_pubtable_publisher tag: {tag}"); return new PubTablePublisher(tag, _publisher_core); } + private void _check_schema_consistency(IBlueTableSchema schema_arg) { + IBlueTableSchema schema; + if (_schema_map.TryGetValue(schema_arg.tableName, out schema)) { + bts.Log.assert(BlueDataUtility.compareSchemas(schema, schema_arg)); + } + } + } } diff --git a/src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs b/src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs index 877101d66..906647b52 100644 --- a/src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs +++ b/src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs @@ -14,7 +14,7 @@ namespace BCG { /// is not uniquely chosen) public ViewPublisher(string name, PubTableBroker br, ViewStream vs) { myName = name; - myView = vs;//String.IsNullOrEmpty(name) ? vs : new ProjectStream(vs, "!", myName); + myView = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name); myBroker = br; myFirstTime = true; myView.onChange += new DataStream.ChangeDelegate(myChangeHandler); diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs index 87db76f18..f6ad09f6e 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs @@ -12,45 +12,58 @@ using BCG.Utilities.General; using System.Linq.Expressions; //using mbts.spark; using mbts; +using BCG.Extensions; -namespace BCG { - public static class BlueDataUtility { +namespace BCG +{ + public static class BlueDataUtility + { #region Extensions - public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) { + public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) + { return BlueDataUtility.get_row_datetime_value(r, col_name, err_value); } - public static DateTime get_datetime(this IBlueRow r, int col_no, DateTime err_value = default(DateTime)) { + public static DateTime get_datetime(this IBlueRow r, int col_no, DateTime err_value = default(DateTime)) + { return BlueDataUtility.get_row_datetime_value(r, col_no, err_value); } - public static T get_enum(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct { + public static T get_enum(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct + { return BlueDataUtility.get_row_enum_value(r, col_name, err_value); } - public static T get_enum(this IBlueRow r, int col_no, T err_value = default(T)) where T : struct { + public static T get_enum(this IBlueRow r, int col_no, T err_value = default(T)) where T : struct + { return BlueDataUtility.get_row_enum_value(r, col_no, err_value); } - public static T get(this IBlueRow r, string col_name, T err_value = default(T)) { + public static T get(this IBlueRow r, string col_name, T err_value = default(T)) + { return get_row_value(r, col_name, err_value); } - public static T get(this IBlueRow r, int col_no, T err_value = default(T)) { + public static T get(this IBlueRow r, int col_no, T err_value = default(T)) + { return get_row_value(r, col_no, err_value); } - public static IEnumerable get_covalues(this IBlueRowCollection rows, string col) { + public static IEnumerable get_covalues(this IBlueRowCollection rows, string col) + { List rv = new List(); foreach (IBlueRow r in rows) rv.Add(r.get(col)); return rv; } - public static IEnumerable get_covalues(IEnumerable rows, string col) { + public static IEnumerable get_covalues(IEnumerable rows, string col) + { List rv = new List(); foreach (IBlueRow r in rows) rv.Add(r.get(col)); return rv; } - public static IBlueRowCollection where(this IBlueRowCollection rows, Func where_func) { + public static IBlueRowCollection where(this IBlueRowCollection rows, Func where_func) + { if (rows == null) return null; - else { + else + { BlueTable rv = new BlueTable(rows.schema); foreach (IBlueRow r in rows) if (where_func(r)) @@ -59,45 +72,71 @@ namespace BCG { } } - public static IEnumerable to_ienum(this IBlueRowCollection rows) { + public static IEnumerable to_ienum(this IBlueRowCollection rows) + { List rv = new List(); foreach (IBlueRow r in rows) rv.Add(r); return rv; } - public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple[] cols) { + public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple[] cols) + { List cis = new List(input_schema.columnInfo); foreach (Tuple c in cols) cis.Add(new ColumnInfo(c.Item2, cis.Count, c.Item1)); return new BlueTableSchema(input_schema.tableName, cis.ToArray()); } - public static IBlueRowCollection to_collection(this IBlueRow r) { + public static IBlueRowCollection to_collection(this IBlueRow r) + { if (r == null) return null; - else { + else + { BlueTable rv = new BlueTable(r.schema); rv.addRow(r); return rv; } } - public static DataPacket build_dp(IBlueRowCollection new_rows, DataPacket last_fired_dp) { + public static DataPacket build_dp(IBlueRowCollection new_rows, DataPacket last_fired_dp) + { IBlueRowCollection added_rows = new_rows; IBlueRowCollection deleted_rows = last_fired_dp == null ? new BlueTable(new_rows.schema) : last_fired_dp.newRows; return new DataPacket(new_rows, added_rows, deleted_rows); } + public static DataPacket build_dp_inc(IBlueRowCollection new_rows, DataPacket last_fired_dp) + { + if (last_fired_dp == null) + { + IBlueRowCollection added_rows = new_rows; + IBlueRowCollection deleted_rows = new BlueTable(new_rows.schema); + return new DataPacket(new_rows, added_rows, deleted_rows); + } + else + { + IBlueRowCollection added_rows = null; + IBlueRowCollection deleted_rows = null; + compute_differences_id_based(new_rows, last_fired_dp.newRows, out deleted_rows, out added_rows); + return new DataPacket(new_rows, added_rows, deleted_rows); + } + } + public static IBlueRowCollection adapt_for_sql_storage(this IBlueRowCollection rows, bool handle_datetime_min = false, - bool handle_double_nan = false) { - if (rows != null) { + bool handle_double_nan = false) + { + if (rows != null) + { BlueTable rv = new BlueTable(rows.schema); - foreach (IBlueRow r in rows) { + foreach (IBlueRow r in rows) + { MutableBlueRow mr = new MutableBlueRow(r); rv.addRow(mr); - foreach (ColumnInfo ci in rows.schema.columnInfo) { + foreach (ColumnInfo ci in rows.schema.columnInfo) + { if (handle_datetime_min && ci.colType == typeof(DateTime) && DateTime.MinValue.Equals(r[ci.colName])) mr.setValue(ci.colName, DBNull.Value); if (handle_double_nan && ci.colType == typeof(double) && Double.IsNaN(r.get(ci.colName))) @@ -105,21 +144,25 @@ namespace BCG { } } return rv; - } else + } + else return null; } #endregion - public static void print_table(this IBlueRowCollection table, Func get_col_width) { + public static void print_table(this IBlueRowCollection table, Func get_col_width) + { var rv = get_printed_rows(table, get_col_width); Console.WriteLine(rv); } - public static string get_printed_rows(this IBlueRowCollection table, Func get_col_width) { + public static string get_printed_rows(this IBlueRowCollection table, Func get_col_width) + { // Print column headers string rv = String.Empty;// = $"{table.schema.tableName}______________________________________"; - foreach (var col_name in table.schema.columnNames) { + foreach (var col_name in table.schema.columnNames) + { int col_width = get_col_width(col_name); rv += $"{col_name.PadRight(col_width)} "; } @@ -127,12 +170,15 @@ namespace BCG { //rv += "______________________________________"; // Print table data - foreach (IBlueRow row in table) { - foreach (string col_name in table.schema.columnNames) { + foreach (IBlueRow row in table) + { + foreach (string col_name in table.schema.columnNames) + { int col_width = get_col_width(col_name); object row_val = row[col_name]; string row_val_str = row_val.ToString(); - if (row_val != null) { + if (row_val != null) + { rv += $"{row_val_str.PadRight(col_width)} "; } } @@ -162,7 +208,8 @@ namespace BCG { { TypeCode.UInt64, typeof(UInt64) } }; - public static IBlueTableSchema form_schema(string table_name, params Tuple[] cols) { + public static IBlueTableSchema form_schema(string table_name, params Tuple[] cols) + { List cis = new List(); foreach (Tuple c in cols) cis.Add(new ColumnInfo(c.Item1, cis.Count, _typecode_to_type[c.Item2])); @@ -178,14 +225,17 @@ namespace BCG { /// /// /// - public static int getRowHashCode(IBlueRow row, bool isColAlphabetically) { + public static int getRowHashCode(IBlueRow row, bool isColAlphabetically) + { string retVal = ""; - if (isColAlphabetically) { + if (isColAlphabetically) + { string[] colNames = row.schema.columnNames; Array.Sort(colNames); foreach (string col in colNames) retVal += row[col].ToString(); - } else + } + else foreach (object o in row) retVal += o.ToString(); return retVal.GetHashCode(); @@ -197,7 +247,8 @@ namespace BCG { /// /// /// - public static int getRowHashCode(IBlueRow row, params string[] colNames) { + public static int getRowHashCode(IBlueRow row, params string[] colNames) + { string retVal = ""; foreach (string colName in colNames) retVal += row[colName].ToString(); @@ -209,7 +260,8 @@ namespace BCG { /// /// /// - public static DataTable toDataTable(IBlueRowCollection rows) { + public static DataTable toDataTable(IBlueRowCollection rows) + { DataTable retTable = getTableFromSchema(rows.schema); retTable.BeginLoadData(); foreach (IBlueRow row in rows) @@ -224,7 +276,8 @@ namespace BCG { /// /// /// - public static DataRow toDataRow(IBlueRow row, DataTable table) { + public static DataRow toDataRow(IBlueRow row, DataTable table) + { DataRow retRow = table.NewRow(); foreach (string colName in row.schema.columnNames) retRow[colName] = row[colName]; @@ -247,7 +300,8 @@ namespace BCG { /// table whos Schema is desired /// /// - public static IBlueTableSchema getSchema(DataTable table) { + public static IBlueTableSchema getSchema(DataTable table) + { ColumnInfo[] cis = new ColumnInfo[table.Columns.Count]; foreach (DataColumn col in table.Columns) cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType); @@ -260,7 +314,8 @@ namespace BCG { /// /// /// - public static IBlueTableSchema getSchema(object o) { + public static IBlueTableSchema getSchema(object o) + { return getSchema(o.GetType()); } @@ -270,7 +325,8 @@ namespace BCG { /// /// Base schema /// Columns to add - public static IBlueTableSchema addColumnsToSchema(IBlueTableSchema schema, params ColumnInfo[] cis) { + public static IBlueTableSchema addColumnsToSchema(IBlueTableSchema schema, params ColumnInfo[] cis) + { ArrayList rv = new ArrayList(schema.columnInfo); rv.AddRange(cis); return new BlueTableSchema(schema.tableName, rv.ToArray(typeof(ColumnInfo)) as ColumnInfo[]); @@ -282,7 +338,8 @@ namespace BCG { /// /// /// - public static IBlueTableSchema appendSchemas(IBlueTableSchema s1, IBlueTableSchema appendedSchema) { + public static IBlueTableSchema appendSchemas(IBlueTableSchema s1, IBlueTableSchema appendedSchema) + { List cis = new List(s1.columnInfo); foreach (ColumnInfo c in appendedSchema.columnInfo) if (c.colName != "id") @@ -291,15 +348,18 @@ namespace BCG { } - public static IBlueTableSchema getSchema(Type t) { + public static IBlueTableSchema getSchema(Type t) + { return getSchema(t, false); } - public static IBlueTableSchema getSchema(Type t, bool createIdField) { + public static IBlueTableSchema getSchema(Type t, bool createIdField) + { return getSchema(t, t.Name, createIdField); } - public static IBlueTableSchema getSchema(Type t, string tableName, bool createIdField) { + public static IBlueTableSchema getSchema(Type t, string tableName, bool createIdField) + { ArrayList colInfos = new ArrayList(); int i = 0; foreach (PropertyInfo pi in t.GetProperties()) @@ -319,17 +379,20 @@ namespace BCG { tableName); } - public static IBlueTableSchema getSchema(Type t, string tableName) { + public static IBlueTableSchema getSchema(Type t, string tableName) + { return new BlueTableSchema(getSchema(t).columnInfo, tableName); } - public static IBlueRow createRowFromType(IBlueTableSchema schema, object o) { + public static IBlueRow createRowFromType(IBlueTableSchema schema, object o) + { return createRowFromType(schema, o, null); } //creates a row from the given type, and if idVal is not null, using the given id - public static IBlueRow createRowFromType(IBlueTableSchema schema, object o, object idVal) { + public static IBlueRow createRowFromType(IBlueTableSchema schema, object o, object idVal) + { object[] rowArray = new object[schema.columnInfo.Length]; Type t = o.GetType(); foreach (PropertyInfo pi in t.GetProperties()) @@ -348,7 +411,8 @@ namespace BCG { } public static MutableBlueRow create_mutable_row(IBlueTableSchema schema, object o, object idVal, - Func type_converter) { + Func type_converter) + { object[] rowArray = new object[schema.columnInfo.Length]; Type t = o.GetType(); foreach (PropertyInfo pi in t.GetProperties()) @@ -367,7 +431,8 @@ namespace BCG { } - public static IBlueTableSchema get_schema(Type t, string tableName, bool createIdField, Func type_converter) { + public static IBlueTableSchema get_schema(Type t, string tableName, bool createIdField, Func type_converter) + { List colInfos = new List(); int i = 0; if (createIdField) @@ -381,13 +446,16 @@ namespace BCG { } public static MutableBlueRow create_mutable_row_not_slow(IBlueTableSchema schema, FieldInfo[] fields, PropertyInfo[] props, - object o, object idVal, Func data_converter) { + object o, object idVal, Func data_converter) + { object[] rowArray = new object[schema.columnInfo.Length]; - if (props != null) { + if (props != null) + { foreach (PropertyInfo pi in props) rowArray[schema[pi.Name].colNo] = data_converter(pi.PropertyType, pi.GetValue(o, null)); } - if (fields != null) { + if (fields != null) + { foreach (FieldInfo fi in fields) rowArray[schema[fi.Name].colNo] = data_converter(fi.FieldType, fi.GetValue(o)); } @@ -396,16 +464,19 @@ namespace BCG { return new MutableBlueRow(schema, rowArray); } - public static T createTypeFromRow(IBlueRow row) { + public static T createTypeFromRow(IBlueRow row) + { return (T)createTypeFromRow(row, typeof(T)); } - public static object createTypeFromRow(IBlueRow row, Type t) { + public static object createTypeFromRow(IBlueRow row, Type t) + { object outValue = Activator.CreateInstance(t); PropertyInfo[] properties = t.GetProperties(); - foreach (PropertyInfo pi in properties) { + foreach (PropertyInfo pi in properties) + { if (myIsActiveProperty(pi)) if (row.Contains(pi.Name)) if (row[pi.Name].Equals(DBNull.Value) && !pi.PropertyType.Equals(typeof(Object))) @@ -414,7 +485,8 @@ namespace BCG { pi.SetValue(outValue, row[pi.Name], null); } FieldInfo[] fields = t.GetFields(); - foreach (FieldInfo fi in fields) { + foreach (FieldInfo fi in fields) + { if (myIsActiveField(fi)) if (row.Contains(fi.Name)) if (row[fi.Name].Equals(DBNull.Value) && !fi.FieldType.Equals(typeof(Object))) @@ -429,15 +501,18 @@ namespace BCG { //these functions should originally have been made public //but changing that would change a bunch of code - public static bool isActiveProperty(PropertyInfo pi) { + public static bool isActiveProperty(PropertyInfo pi) + { return myIsActiveProperty(pi); } - public static bool isActiveField(FieldInfo fi) { + public static bool isActiveField(FieldInfo fi) + { return myIsActiveField(fi); } - private static bool myIsActiveProperty(PropertyInfo pi) { + private static bool myIsActiveProperty(PropertyInfo pi) + { if (pi.GetGetMethod() == null) return false; foreach (Attribute att in pi.GetCustomAttributes(true)) @@ -446,7 +521,8 @@ namespace BCG { return true; } - private static bool myIsActiveField(FieldInfo fi) { + private static bool myIsActiveField(FieldInfo fi) + { if (!fi.IsPublic) return false; foreach (Attribute att in fi.GetCustomAttributes(true)) @@ -455,7 +531,8 @@ namespace BCG { return true; } - private static bool myIsActiveMember(MemberInfo mi) { + private static bool myIsActiveMember(MemberInfo mi) + { if (!(mi is FieldInfo) || !(mi is PropertyInfo) || (mi is PropertyInfo && (mi as PropertyInfo).GetGetMethod() == null) || @@ -467,17 +544,21 @@ namespace BCG { return true; } - private static bool myDoesRowExistsInTable(IBlueRowCollection brc2, string[] columns, object[] oarray) { + private static bool myDoesRowExistsInTable(IBlueRowCollection brc2, string[] columns, object[] oarray) + { bool rowExists = false; - foreach (IBlueRow br in brc2) { + foreach (IBlueRow br in brc2) + { bool allElementsSame = true; - for (int i = 0; i < columns.Length; i++) { + for (int i = 0; i < columns.Length; i++) + { string col = columns[i]; object o1 = br[col]; object o2 = oarray[i]; allElementsSame = (o1.Equals(o2) == false) ? false : allElementsSame; } - if (allElementsSame == true) { + if (allElementsSame == true) + { rowExists = true; break; } @@ -492,7 +573,8 @@ namespace BCG { /// Schema whose table needs to be created /// /// - public static DataTable getTableFromSchema(IBlueTableSchema schema) { + public static DataTable getTableFromSchema(IBlueTableSchema schema) + { DataTable retTable = new DataTable(schema.tableName); foreach (ColumnInfo ci in schema.columnInfo) retTable.Columns.Add(ci.colName, ci.colType); @@ -504,20 +586,24 @@ namespace BCG { /// /// source row /// - public static IBlueRow toBlueRow(DataRow row) { + public static IBlueRow toBlueRow(DataRow row) + { return new BlueRow(getSchema(row), row); } - public static IBlueRowCollection toIBlueRowCollection(DataTable dt) { + public static IBlueRowCollection toIBlueRowCollection(DataTable dt) + { BlueTable retVal = new BlueTable(getSchema(dt)); foreach (DataRow dr in dt.Rows) retVal.addRow(dr); return retVal; } - public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) { + public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) + { BlueTable retVal = null; - foreach (DataRow dr in rows) { + foreach (DataRow dr in rows) + { IBlueRow r = toBlueRow(dr); if (retVal == null) retVal = new BlueTable(r.schema); @@ -534,7 +620,8 @@ namespace BCG { /// /// /// - public static IBlueRow extractRow(IBlueRowCollection rows, int index) { + public static IBlueRow extractRow(IBlueRowCollection rows, int index) + { int i = 0; foreach (IBlueRow row in rows) if (i++ == index) @@ -548,13 +635,15 @@ namespace BCG { /// /// /// - public static bool computeRowEquality(IBlueRow r1, IBlueRow r2) { + public static bool computeRowEquality(IBlueRow r1, IBlueRow r2) + { return computeRowEquality(r1, r2, r1.schema.columnNames); } public static bool computeRowEquality(IBlueRow r1, IBlueRow r2, - params string[] colNames) { + params string[] colNames) + { return compareRows(r1, r2, colNames) == 0 ? true : false; } @@ -565,7 +654,8 @@ namespace BCG { /// /// /// - public static string rowToString(IBlueRow row, params string[] colNames) { + public static string rowToString(IBlueRow row, params string[] colNames) + { string retVal = ""; foreach (string colName in colNames) retVal += (row[colName] != null ? row[colName].ToString().Trim() : "NULL") + "\n"; @@ -574,10 +664,12 @@ namespace BCG { public static string rowToString(IBlueRow row) { return rowToString(row, row.schema.columnNames); } - public static string rowCollectionToString(IBlueRowCollection rowColl) { + public static string rowCollectionToString(IBlueRowCollection rowColl) + { if (rowColl == null) return "Null IBlueRowCollection"; - else { + else + { string retVal = ""; foreach (IBlueRow r in rowColl) retVal += retVal == "" ? rowToString(r) : "\n" + rowToString(r); @@ -596,9 +688,11 @@ namespace BCG { /// public static int compareRows(IBlueRow r1, IBlueRow r2, - params string[] colNames) { + params string[] colNames) + { int retVal = 0; - foreach (string colName in colNames) { + foreach (string colName in colNames) + { object val1 = r1[colName]; object val2 = r2[colName]; retVal = _compare_row_cols(val1, val2); @@ -608,13 +702,16 @@ namespace BCG { return retVal; } - private static int _compare_row_cols(object v1, object v2) { + private static int _compare_row_cols(object v1, object v2) + { if (_is_null(v1)) return _is_null(v2) ? 0 : -1; - else { + else + { if (_is_null(v2)) return 1; - else { + else + { if (v1 is IComparable && v2 is IComparable) return (v1 as IComparable).CompareTo(v2); else if (v1 is byte[] && v2 is byte[]) @@ -625,7 +722,8 @@ namespace BCG { } } - private static bool _compare_bytes(byte[] v1, byte[] v2) { + private static bool _compare_bytes(byte[] v1, byte[] v2) + { if (v1.Length != v2.Length) return false; else @@ -642,7 +740,8 @@ namespace BCG { /// a row with the values to be converted /// the desired schema for the output row /// - public static IBlueRow changeSchema(IBlueRow inRow, IBlueTableSchema newSchema) { + public static IBlueRow changeSchema(IBlueRow inRow, IBlueTableSchema newSchema) + { object[] outputValues = new object[newSchema.columnInfo.Length]; for (int i = 0; i < outputValues.Length; ++i) @@ -657,7 +756,8 @@ namespace BCG { /// /// /// - public static IBlueRowCollection combineTables(params IBlueRowCollection[] tables) { + public static IBlueRowCollection combineTables(params IBlueRowCollection[] tables) + { //get some base cases out of the way if (tables.Length == 0) return null; @@ -669,12 +769,14 @@ namespace BCG { int numCols = tables[0].schema.columnInfo.Length; int idCol = tables[0].schema["id"].colNo; IBlueRowCollection previousTable = null; - foreach (IBlueRowCollection table in tables) { + foreach (IBlueRowCollection table in tables) + { if (previousTable != null) if (!compareSchemas(previousTable.schema, table.schema)) throw new ArgumentException("Error, tables must have the same schema to be combined"); - foreach (IBlueRow r in table) { + foreach (IBlueRow r in table) + { object[] values = new object[numCols]; r.CopyTo(values, 0); values[idCol] = idToUseNext++; @@ -694,7 +796,8 @@ namespace BCG { /// /// /// - public static string[] compareRows(IBlueRow r1, IBlueRow r2) { + public static string[] compareRows(IBlueRow r1, IBlueRow r2) + { ArrayList retVal = new ArrayList(); foreach (string col in r1.schema.columnNames) if (!r1[col].Equals(r2[col])) @@ -702,7 +805,8 @@ namespace BCG { return retVal.ToArray(typeof(string)) as string[]; } - public static BlueTable cloneRowCollection(IBlueRowCollection rows) { + public static BlueTable cloneRowCollection(IBlueRowCollection rows) + { BlueTable retVal = new BlueTable(rows.schema); foreach (IBlueRow r in rows) retVal.addRow(new BlueRow(r)); @@ -717,7 +821,8 @@ namespace BCG { /// /// /// - public static IBlueRow getNewReplacedRow(IBlueRow r, object[] newValues, int[] indexes) { + public static IBlueRow getNewReplacedRow(IBlueRow r, object[] newValues, int[] indexes) + { if (!newValues.Length.Equals(indexes.Length)) throw new Exception("Debug Assert"); @@ -728,11 +833,13 @@ namespace BCG { return new BlueRow(r.schema, rowArray); } - public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, int index) { + public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, int index) + { return getNewReplacedRow(r, new object[] { newValue }, new int[] { index }); } - public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, string colName) { + public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, string colName) + { return getNewReplacedRow(r, newValue, r.schema[colName].colNo); } @@ -742,7 +849,8 @@ namespace BCG { /// /// /// - public static IBlueRow computeMaxRow(IBlueRowCollection rows, params string[] cols) { + public static IBlueRow computeMaxRow(IBlueRowCollection rows, params string[] cols) + { IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null; foreach (IBlueRow r in rows) retVal = compareRows(r, retVal, cols) > 0 ? r : retVal; @@ -755,7 +863,8 @@ namespace BCG { /// /// /// - public static IBlueRow computeMinRow(IBlueRowCollection rows, params string[] cols) { + public static IBlueRow computeMinRow(IBlueRowCollection rows, params string[] cols) + { IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null; foreach (IBlueRow r in rows) retVal = compareRows(r, retVal, cols) < 0 ? r : retVal; @@ -768,20 +877,24 @@ namespace BCG { /// /// /// - public static IBlueRow computeSum(IBlueRowCollection rows, params string[] cols) { + public static IBlueRow computeSum(IBlueRowCollection rows, params string[] cols) + { if (rows.Count == 0) return null; IBlueRow retVal = BlueDataUtility.extractRow(rows, 0); object[] sums = new object[cols.Length]; int[] idxs = new int[cols.Length]; - for (int i = 0; i < cols.Length; i++) { + for (int i = 0; i < cols.Length; i++) + { sums[i] = 0.0; idxs[i] = rows.schema[cols[i]].colNo; } - foreach (IBlueRow r in rows) { - for (int i = 0; i < cols.Length; i++) { + foreach (IBlueRow r in rows) + { + for (int i = 0; i < cols.Length; i++) + { double dval = BlueDataUtility.get_row_value(r, cols[i], 0.0); //r[cols[i]] == null || r[cols[i]] is DBNull ? 0.0 : (double)r[cols[i]]; sums[i] = (double)sums[i] + dval; @@ -792,7 +905,8 @@ namespace BCG { } - public static IBlueRow computeSum(IBlueRowCollection rows, params string[] cols) { + public static IBlueRow computeSum(IBlueRowCollection rows, params string[] cols) + { if (rows.Count == 0) return null; IBlueRow retVal = BlueDataUtility.extractRow(rows, 0); if (rows.Count == 1) return retVal; @@ -800,13 +914,16 @@ namespace BCG { object[] sums = new object[cols.Length]; int[] idxs = new int[cols.Length]; - for (int i = 0; i < cols.Length; i++) { + for (int i = 0; i < cols.Length; i++) + { sums[i] = default(T); idxs[i] = rows.schema[cols[i]].colNo; } - foreach (IBlueRow r in rows) { - for (int i = 0; i < cols.Length; i++) { + foreach (IBlueRow r in rows) + { + for (int i = 0; i < cols.Length; i++) + { T tval = r[cols[i]] == null || r[cols[i]] is DBNull ? default(T) : (T)r[cols[i]]; sums[i] = MagicAdder.Add((T)sums[i], tval); } @@ -821,14 +938,16 @@ namespace BCG { /// /// /// - public static bool rowPresent(IBlueRowCollection table, IBlueRow row) { + public static bool rowPresent(IBlueRowCollection table, IBlueRow row) + { return table.containsID((int)row["id"]); } public static void computeDifferences(IBlueRowCollection newRows, IBlueRowCollection oldRows, out IBlueRowCollection dRows, - out IBlueRowCollection aRows) { + out IBlueRowCollection aRows) + { computeDifferences( newRows, oldRows, @@ -838,16 +957,26 @@ namespace BCG { } public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows, - out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) { + out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) + { int[] col_indices = new int[new_rows.schema.columnNames.Length]; for (int i = 0; i < col_indices.Length; i++) col_indices[i] = i; compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows); } + public static void compute_differences_id_based(IBlueRowCollection new_rows, IBlueRowCollection old_rows, + out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) + { + int[] col_indices = new int[] { new_rows.schema["id"].colNo }; + compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows); + } + public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows, int[] col_indices, - out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) { - Func rows_equal = (r1, r2) => { + out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) + { + Func rows_equal = (r1, r2) => + { foreach (int col_no in col_indices) if (!(r1[col_no] ?? DBNull.Value).Equals(r2[col_no] ?? DBNull.Value)) return false; @@ -859,7 +988,8 @@ namespace BCG { foreach (IBlueRow r in new_rows) if (!BlueDataUtility.rowPresent(old_rows, r)) add_rows.addRow(r); - else if (!rows_equal(r, old_rows[(int)r["id"]])) { + else if (!rows_equal(r, old_rows[(int)r["id"]])) + { add_rows.addRow(r); del_rows.addRow(old_rows[(int)r["id"]]); } @@ -876,7 +1006,8 @@ namespace BCG { IBlueRowCollection c2, string[] columns, out IBlueRowCollection dRows, - out IBlueRowCollection aRows) { + out IBlueRowCollection aRows) + { dRows = new BlueTable(c1.schema); aRows = new BlueTable(c1.schema); @@ -885,7 +1016,8 @@ namespace BCG { if (!BlueDataUtility.rowPresent(c2, nr)) (aRows as BlueTable).addRow(nr); else if (BlueDataUtility.compareRows(nr, c2[(int) - nr["id"]], columns) != 0) { + nr["id"]], columns) != 0) + { (aRows as BlueTable).addRow(nr); (dRows as BlueTable).addRow(c2[(int)nr["id"]]); @@ -898,11 +1030,14 @@ namespace BCG { } //brcDiff = brc1 - brc2 - public static void diff(IBlueRowCollection brc1, IBlueRowCollection brc2, string[] columns, out IBlueRowCollection brcDiff) { + public static void diff(IBlueRowCollection brc1, IBlueRowCollection brc2, string[] columns, out IBlueRowCollection brcDiff) + { brcDiff = new BlueTable(brc1.schema); - foreach (IBlueRow br in brc1) { + foreach (IBlueRow br in brc1) + { object[] oarray = new object[columns.Length]; - for (int i = 0; i < columns.Length; i++) { + for (int i = 0; i < columns.Length; i++) + { string col = columns[i]; oarray[i] = br[col]; } @@ -917,7 +1052,8 @@ namespace BCG { /// /// /// - public static void absorbChanges(BlueTable t, IBlueRowCollection dels, IBlueRowCollection adds) { + public static void absorbChanges(BlueTable t, IBlueRowCollection dels, IBlueRowCollection adds) + { foreach (IBlueRow r in dels) if (!t.deleteRow(r)) throw new Exception(String.Format("{0} does not have row with id {1} to delete", t.schema.tableName, r["id"])); @@ -926,7 +1062,8 @@ namespace BCG { t.addRow(r); } - public static IBlueRowCollection toBlueRowCollection(IBlueRow r) { + public static IBlueRowCollection toBlueRowCollection(IBlueRow r) + { BlueTable rv = new BlueTable(r.schema); rv.addRow(r); return rv; @@ -938,9 +1075,11 @@ namespace BCG { /// /// /// - public static IBlueRowCollection toBlueRowCollection(ICollection col) { + public static IBlueRowCollection toBlueRowCollection(ICollection col) + { BlueTable retVal = null; - foreach (IBlueRow r in col) { + foreach (IBlueRow r in col) + { retVal = retVal ?? new BlueTable(r.schema); retVal.addRow(r); } @@ -959,19 +1098,23 @@ namespace BCG { IBlueRowCollection adds, out IBlueRowCollection truelyDels, out IBlueRowCollection truelyAdds, - string[] cols) { + string[] cols) + { truelyDels = new BlueTable(dels); truelyAdds = new BlueTable(adds); - foreach (IBlueRow r in dels) { + foreach (IBlueRow r in dels) + { if (adds.containsID((int)r["id"]) && - BlueDataUtility.computeRowEquality(adds[(int)r["id"]], r, cols)) { + BlueDataUtility.computeRowEquality(adds[(int)r["id"]], r, cols)) + { (truelyDels as BlueTable).deleteRow((int)r["id"]); (truelyAdds as BlueTable).deleteRow((int)r["id"]); } } } - public static IBlueRowCollection getSerializableCollection(IBlueRowCollection coll) { + public static IBlueRowCollection getSerializableCollection(IBlueRowCollection coll) + { BlueTable retVal = new OrderedBlueTable(coll.schema); foreach (IBlueRow r in coll) retVal.addRow(new BlueRow(r)); @@ -984,7 +1127,8 @@ namespace BCG { /// /// /// - public static ArrayList extractColumn(IBlueRowCollection rows, string colName) { + public static ArrayList extractColumn(IBlueRowCollection rows, string colName) + { ArrayList rv = new ArrayList(); foreach (IBlueRow r in rows) rv.Add(r[colName]); @@ -1003,16 +1147,19 @@ namespace BCG { public static void checkConsistency(IBlueRowCollection oldState, IBlueRowCollection newRows, IBlueRowCollection delRows, - IBlueRowCollection addedRows) { + IBlueRowCollection addedRows) + { BlueTable calc = new BlueTable(oldState); Debug.Assert(oldState != null && newRows != null && delRows != null && addedRows != null); - foreach (IBlueRow r in delRows) { + foreach (IBlueRow r in delRows) + { Debug.Assert(calc.containsID((int)r["id"]), "deleted row's id = " + r["id"] + " does not exist"); calc.deleteRow(r); } - foreach (IBlueRow r in addedRows) { + foreach (IBlueRow r in addedRows) + { Debug.Assert(!calc.containsID((int)r["id"]), "added rows's id = " + r["id"].ToString() + " already exists"); calc.addRow(r); } @@ -1029,7 +1176,8 @@ namespace BCG { /// /// /// - public static object[] getSelectedValues(IBlueRow r, params string[] colNames) { + public static object[] getSelectedValues(IBlueRow r, params string[] colNames) + { object[] retVal = new object[colNames.Length]; for (int i = 0; i < colNames.Length; i++) retVal[i] = r[colNames[i]]; @@ -1045,8 +1193,10 @@ namespace BCG { /// /// /// - public static IBlueRow getRowWithKey(IBlueRowCollection rows, string[] keyColumns, object[] colValues) { - foreach (IBlueRow r in rows) { + public static IBlueRow getRowWithKey(IBlueRowCollection rows, string[] keyColumns, object[] colValues) + { + foreach (IBlueRow r in rows) + { object[] vals = getSelectedValues(r, keyColumns); if (CollectionHelpers.equal(vals, colValues)) return r; @@ -1054,7 +1204,8 @@ namespace BCG { return null; } - public static IBlueRowCollection deepCopy(IBlueRowCollection rows) { + public static IBlueRowCollection deepCopy(IBlueRowCollection rows) + { BlueTable retVal = new BlueTable(rows.schema); foreach (IBlueRow r in rows) retVal.addRow(new BlueRow(r)); @@ -1068,7 +1219,8 @@ namespace BCG { /// /// /// - public static int[] getColumnsInSortedOrder(IBlueTableSchema schema, bool includeId) { + public static int[] getColumnsInSortedOrder(IBlueTableSchema schema, bool includeId) + { ArrayList names = new ArrayList(schema.columnNames), retVal = new ArrayList(); names.Sort(); @@ -1081,33 +1233,40 @@ namespace BCG { //this class is used so that we can sort a schema by column name //that way, prepublished tables will allways get published with schemas in the same order //regardless of how the underlying streams end up working - private class ColumnInfoComparisonWrapper : IComparable { + private class ColumnInfoComparisonWrapper : IComparable + { ColumnInfo myColumn; - public string ColumnName { + public string ColumnName + { get { return myColumn.colName; } //can't set it, these are only for reading columns } - public ColumnInfo Column { + public ColumnInfo Column + { get { return myColumn; } } - public ColumnInfoComparisonWrapper(ColumnInfo theColumn) { + public ColumnInfoComparisonWrapper(ColumnInfo theColumn) + { myColumn = theColumn; } - public int CompareTo(ColumnInfoComparisonWrapper otherGuy) { + public int CompareTo(ColumnInfoComparisonWrapper otherGuy) + { return ColumnName.CompareTo(otherGuy.ColumnName); } } - public static IBlueTableSchema getSortedSchema(IBlueTableSchema inSchema) { + public static IBlueTableSchema getSortedSchema(IBlueTableSchema inSchema) + { List sortedColNames = new List(inSchema.columnNames); sortedColNames.Sort(); List sortedColInfos = new List(); int colNo = 0; - foreach (string c in sortedColNames) { + foreach (string c in sortedColNames) + { ColumnInfo ci = inSchema[c]; sortedColInfos.Add(new ColumnInfo(c, colNo++, ci.colType)); @@ -1115,13 +1274,15 @@ namespace BCG { return new BlueTableSchema(inSchema.tableName, sortedColInfos.ToArray()); } - public static IBlueTableSchema getSortedSchema(IBlueTableSchema theSchema, out int[] sortedSchemaToOriginalSchemaMapping, out int idColumnInSortedSchema) { + public static IBlueTableSchema getSortedSchema(IBlueTableSchema theSchema, out int[] sortedSchemaToOriginalSchemaMapping, out int idColumnInSortedSchema) + { //for now this function will not do anything //that is because we don't want to break the C++ deserialization code int numCols = theSchema.columnNames.Length; sortedSchemaToOriginalSchemaMapping = new int[numCols]; - for (int col = 0; col < theSchema.columnNames.Length; ++col) { + for (int col = 0; col < theSchema.columnNames.Length; ++col) + { //the mapping is just the identity mapping sortedSchemaToOriginalSchemaMapping[col] = col; @@ -1175,34 +1336,62 @@ namespace BCG { /// /// /// true if equal - public static bool compareSchemas(IBlueTableSchema s1, IBlueTableSchema s2) { + public static bool compareSchemas(IBlueTableSchema s1, IBlueTableSchema s2) + { bool result = true; - try { + try + { if (s1.columnInfo.Length != s2.columnInfo.Length) result = false; else - foreach (ColumnInfo ci1 in s1.columnInfo) { + foreach (ColumnInfo ci1 in s1.columnInfo) + { ColumnInfo ci2 = s2[ci1.colNo]; - if (ci1.colName != ci2.colName || ci1.colType != ci2.colType) { + if (ci1.colName != ci2.colName || ci1.colType != ci2.colType || ci1.colNo != ci2.colNo) + { result = false; break; } } - } catch { + } + catch + { result = false; // the col# wasn't found in s2 } return result; } - public static IBlueRowCollection removeRows(BlueTable table, string[] cols, object[] vals) { + public static void validate_row_schema(IBlueRow r) + { + foreach (var ci in r.schema.columnInfo) + { + object rval = r[ci.colName]; + if (rval == null) + continue; + bts.Log.assert(rval.isDBNull() || rval.GetType().Equals(ci.colType)); + } + } + + public static void validate_table_schema(IBlueRowCollection rows) + { + foreach (IBlueRow r in rows) + { + validate_row_schema(r); + } + } + + public static IBlueRowCollection removeRows(BlueTable table, string[] cols, object[] vals) + { IBlueRowCollection dels = findMatchingRows(table, cols, vals); table.deleteRows(dels); return dels; } - public static IBlueRowCollection findMatchingRows(IBlueRowCollection rows, string[] cols, object[] vals) { + public static IBlueRowCollection findMatchingRows(IBlueRowCollection rows, string[] cols, object[] vals) + { BlueTable rv = new BlueTable(rows.schema); - foreach (IBlueRow r in rows) { + foreach (IBlueRow r in rows) + { bool match = true; for (int i = 0; i < cols.Length; i++) match &= vals[i].Equals(r[cols[i]]); @@ -1212,7 +1401,8 @@ namespace BCG { return rv; } - public static void checkRowTypes(IBlueRowCollection rows) { + public static void checkRowTypes(IBlueRowCollection rows) + { foreach (IBlueRow r in rows) foreach (string col in r.schema.columnNames) if (r[col] != null && r[col] != DBNull.Value) @@ -1220,12 +1410,14 @@ namespace BCG { String.Format("col {0} in table {1} is of type {2} in schema but is a {3} in the row", col, rows.schema.tableName, r.schema[col].colType.ToString(), r[col].GetType().ToString())); } - public static void addRows(ref BlueTable table, ICollection rows) { + public static void addRows(ref BlueTable table, ICollection rows) + { foreach (IBlueRow r in rows) table.addRow(r); } - public static void removeRows(ref BlueTable table, ICollection rows) { + public static void removeRows(ref BlueTable table, ICollection rows) + { foreach (IBlueRow r in rows) table.deleteRow(r); } @@ -1233,7 +1425,8 @@ namespace BCG { public static void sortOutPacket(IBlueRowCollection dels, IBlueRowCollection adds, string[] keyCols, out IBlueRowCollection addedRows, out IBlueRowCollection deletedRows, - out IBlueRowCollection changedRows) { + out IBlueRowCollection changedRows) + { addedRows = new BlueTable(dels.schema); deletedRows = new BlueTable(dels.schema); changedRows = new BlueTable(dels.schema); @@ -1246,39 +1439,49 @@ namespace BCG { foreach (IBlueRow r in adds) keyedAdds.update(r); - foreach (IBlueRow r in keyedDels) { + foreach (IBlueRow r in keyedDels) + { object[] key = BlueDataUtility.getSelectedValues(r, keyCols); - if (keyedAdds.getRow(key) != null) { + if (keyedAdds.getRow(key) != null) + { ///change (changedRows as BlueTable).addRow(r); - } else { + } + else + { ///delete (deletedRows as BlueTable).addRow(r); } } - foreach (IBlueRow r in keyedAdds) { + foreach (IBlueRow r in keyedAdds) + { object[] key = BlueDataUtility.getSelectedValues(r, keyCols); - if (keyedDels.getRow(key) == null) { + if (keyedDels.getRow(key) == null) + { (addedRows as BlueTable).addRow(r); } } } - public static int HashExpStrike(DateTime exp, double strike) { + public static int HashExpStrike(DateTime exp, double strike) + { return ((DateTime)exp).ToShortDateString().GetHashCode() ^ strike.GetHashCode(); } - public static IEnumerable collection_to_ienum(IBlueRowCollection rows) { + public static IEnumerable collection_to_ienum(IBlueRowCollection rows) + { List rv = new List(); foreach (IBlueRow r in rows) rv.Add(r); return rv; } - public static IBlueRowCollection ienum_to_collection(IEnumerable rows) { + public static IBlueRowCollection ienum_to_collection(IEnumerable rows) + { BlueTable rv = null; - if (rows.Count() > 0) { + if (rows.Count() > 0) + { rv = new BlueTable(rows.ElementAt(0).schema); foreach (IBlueRow r in rows) rv.addRow(r); @@ -1286,13 +1489,15 @@ namespace BCG { return rv; } - public static IBlueTableSchema add_column(IBlueTableSchema input_schema, string new_col_name, Type new_col_type) { + public static IBlueTableSchema add_column(IBlueTableSchema input_schema, string new_col_name, Type new_col_type) + { List cis = new List(input_schema.columnInfo); cis.Add(new ColumnInfo(new_col_name, cis.Count, new_col_type)); return new BlueTableSchema(input_schema.tableName, cis.ToArray()); } - public static IBlueTableSchema remove_columns(IBlueTableSchema input_schema, params string[] col_names) { + public static IBlueTableSchema remove_columns(IBlueTableSchema input_schema, params string[] col_names) + { Dictionary removed_col_names = new Dictionary(); foreach (string c in col_names) removed_col_names[c] = true; @@ -1303,14 +1508,16 @@ namespace BCG { return new BlueTableSchema(cis.ToArray(), input_schema.tableName); } - public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) { + public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) + { List rv = new List(); foreach (string c in col_names) rv.Add(schema[c].colNo); return rv.ToArray(); } - public static string get_appended_string_val(IBlueRow r, int[] col_indices) { + public static string get_appended_string_val(IBlueRow r, int[] col_indices) + { System.Text.StringBuilder sb = new System.Text.StringBuilder(); for (int i = 0; i < col_indices.Length; i++) sb.Append(r[col_indices[i]]); @@ -1318,29 +1525,35 @@ namespace BCG { } public delegate string RowValueToString(IBlueRow r, int col_index); - public static string get_appended_string_val(IBlueRow r, int[] col_indices, RowValueToString to_string_conv) { + public static string get_appended_string_val(IBlueRow r, int[] col_indices, RowValueToString to_string_conv) + { System.Text.StringBuilder sb = new System.Text.StringBuilder(); for (int i = 0; i < col_indices.Length; i++) sb.Append(to_string_conv(r, i)); return sb.ToString(); } - public static IDictionary> group_together(IBlueRowCollection rows, string[] group_by_col_names) { + public static IDictionary> group_together(IBlueRowCollection rows, string[] group_by_col_names) + { BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length); return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher); } - public static IDictionary> group_together(IBlueRowCollection rows, string[] group_by_col_names, ref BlueRowHasher hasher) { + public static IDictionary> group_together(IBlueRowCollection rows, string[] group_by_col_names, ref BlueRowHasher hasher) + { return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher); } - public static IDictionary> group_together(IBlueRowCollection rows, int[] group_by_col_nos, ref BlueRowHasher hasher) { + public static IDictionary> group_together(IBlueRowCollection rows, int[] group_by_col_nos, ref BlueRowHasher hasher) + { Dictionary> rv = new Dictionary>(); hasher = hasher ?? new BlueRowHasher(group_by_col_nos.Length); - foreach (IBlueRow r in rows) { + foreach (IBlueRow r in rows) + { int row_hash = hasher.get_hash(r, group_by_col_nos); List this_group_rows = null; - if (!rv.TryGetValue(row_hash, out this_group_rows)) { + if (!rv.TryGetValue(row_hash, out this_group_rows)) + { this_group_rows = new List(); rv.Add(row_hash, this_group_rows); } @@ -1349,7 +1562,8 @@ namespace BCG { return rv; } - public static void copy_to_row(IBlueRow row, string[] data_col_names, MutableBlueRow output_row) { + public static void copy_to_row(IBlueRow row, string[] data_col_names, MutableBlueRow output_row) + { foreach (string cn in data_col_names) output_row.setValue(cn, row[cn]); } @@ -1358,52 +1572,64 @@ namespace BCG { /// Copies the column nos specified in data_cols from row, to output_row /// starting from index start_col_index /// - public static void copy_to_row(IBlueRow row, int[] data_col_nos, object[] output_row, int start_col_index) { + public static void copy_to_row(IBlueRow row, int[] data_col_nos, object[] output_row, int start_col_index) + { foreach (int col_no in data_col_nos) output_row[start_col_index++] = row[col_no]; } - public static object compute_min_max(object v1, object v2, bool is_max) { - if (!TypeHelper.is_dbnull(v1) && !TypeHelper.is_dbnull(v2)) { - if (TypeHelper.is_numeric_type(v1) && TypeHelper.is_numeric_type(v2)) { + public static object compute_min_max(object v1, object v2, bool is_max) + { + if (!TypeHelper.is_dbnull(v1) && !TypeHelper.is_dbnull(v2)) + { + if (TypeHelper.is_numeric_type(v1) && TypeHelper.is_numeric_type(v2)) + { double v1_num = (double)Convert.ChangeType(v1, TypeCode.Double), v2_num = (double)Convert.ChangeType(v2, TypeCode.Double); v1_num = Double.IsNaN(v1_num) ? (is_max ? Double.MinValue : Double.MaxValue) : v1_num; v2_num = Double.IsNaN(v2_num) ? (is_max ? Double.MinValue : Double.MaxValue) : v2_num; double rv = is_max ? Math.Max(v1_num, v2_num) : Math.Min(v1_num, v2_num); return Convert.ChangeType(rv, v1.GetType()); - } else + } + else return v1; - } else + } + else return TypeHelper.is_dbnull(v1) ? v2 : v1; } #region Sorting - public static class ParallelSort { - public class TreeRowEffComparer : IComparer { + public static class ParallelSort + { + public class TreeRowEffComparer : IComparer + { private int _depth_in_tree_col_no; private int[] _aggregation_tree_col_nos; private Pair[] _sort_map; - public TreeRowEffComparer(int depth_in_tree_col_no, int[] aggregation_tree_col_nos, Pair[] sort_map) { + public TreeRowEffComparer(int depth_in_tree_col_no, int[] aggregation_tree_col_nos, Pair[] sort_map) + { _depth_in_tree_col_no = depth_in_tree_col_no; _aggregation_tree_col_nos = aggregation_tree_col_nos; _sort_map = sort_map; } - public int Compare(IBlueRow x, IBlueRow y) { + public int Compare(IBlueRow x, IBlueRow y) + { return BlueDataUtility.sort_in_a_tree(x, y, _depth_in_tree_col_no, _aggregation_tree_col_nos, _sort_map); } } - public class TabularRowComparer : IComparer { + public class TabularRowComparer : IComparer + { private IEnumerable> _sort_map; public TabularRowComparer(IEnumerable> sort_map) { _sort_map = sort_map; } public int Compare(IBlueRow x, IBlueRow y) { return BlueDataUtility.compare_rows(x, y, _sort_map); } } - public class TabularRowHashPairComparer : IComparer> { + public class TabularRowHashPairComparer : IComparer> + { private Pair[] _sort_map; public TabularRowHashPairComparer(Pair[] sort_map) { _sort_map = sort_map; } @@ -1412,11 +1638,13 @@ namespace BCG { #region Public Static Methods - public static void QuicksortSequential(IList arr, IComparer comparer) { + public static void QuicksortSequential(IList arr, IComparer comparer) + { QuicksortSequential(arr, comparer, 0, arr.Count - 1); } - public static void QuicksortParallel(IList arr, IComparer comparer) { + public static void QuicksortParallel(IList arr, IComparer comparer) + { QuicksortParallel(arr, comparer, 0, arr.Count - 1); } @@ -1424,20 +1652,27 @@ namespace BCG { #region Private Static Methods - private static void QuicksortSequential(IList arr, IComparer comparer, int left, int right) { - if (right > left) { + private static void QuicksortSequential(IList arr, IComparer comparer, int left, int right) + { + if (right > left) + { int pivot = Partition(arr, comparer, left, right); QuicksortSequential(arr, comparer, left, pivot - 1); QuicksortSequential(arr, comparer, pivot + 1, right); } } - private static void QuicksortParallel(IList arr, IComparer comparer, int left, int right) { + private static void QuicksortParallel(IList arr, IComparer comparer, int left, int right) + { const int SEQUENTIAL_THRESHOLD = 64; - if (right > left) { - if (right - left < SEQUENTIAL_THRESHOLD) { + if (right > left) + { + if (right - left < SEQUENTIAL_THRESHOLD) + { QuicksortSequential(arr, comparer, left, right); - } else { + } + else + { int pivot = Partition(arr, comparer, left, right); System.Threading.Tasks.Parallel.Invoke(new Action[] { delegate {QuicksortParallel(arr, comparer, left, pivot - 1);}, delegate {QuicksortParallel(arr, comparer, pivot + 1, right); } @@ -1446,21 +1681,25 @@ namespace BCG { } } - private static void Swap(IList arr, int i, int j) { + private static void Swap(IList arr, int i, int j) + { T tmp = arr[i]; arr[i] = arr[j]; arr[j] = tmp; } - private static int Partition(IList arr, IComparer comparer, int low, int high) { + private static int Partition(IList arr, IComparer comparer, int low, int high) + { // Simple partitioning implementation int pivotPos = (high + low) / 2; T pivot = arr[pivotPos]; Swap(arr, low, pivotPos); int left = low; - for (int i = low + 1; i <= high; i++) { - if (comparer.Compare(arr[i], pivot) < 0) { + for (int i = low + 1; i <= high; i++) + { + if (comparer.Compare(arr[i], pivot) < 0) + { left++; Swap(arr, i, left); } @@ -1473,8 +1712,10 @@ namespace BCG { #endregion } - public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary sort_map) { - foreach (KeyValuePair p in sort_map) { + public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary sort_map) + { + foreach (KeyValuePair p in sort_map) + { int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value); if (comp != 0) return comp; @@ -1482,8 +1723,10 @@ namespace BCG { return 0; } - public static int compare_rows(IBlueRow r1, IBlueRow r2, IEnumerable> sort_map) { - foreach (Pair p in sort_map) { + public static int compare_rows(IBlueRow r1, IBlueRow r2, IEnumerable> sort_map) + { + foreach (Pair p in sort_map) + { int comp = compare_send_dontcare_to_last(r1, p.first, r2, p.first, p.second); if (comp != 0) return comp; @@ -1493,13 +1736,15 @@ namespace BCG { public static int compare_send_dontcare_to_last(IBlueRow r1, int r1_col_no, IBlueRow r2, int r2_col_no, - bool low_to_high) { + bool low_to_high) + { object o1 = BlueDataUtility.get_row_value(r1, r1_col_no, DBNull.Value), o2 = BlueDataUtility.get_row_value(r2, r2_col_no, DBNull.Value); return compare_send_dontcare_to_last(o1, o2, low_to_high); } - public static List> get_indexed_sort_map(List> sort_map, IBlueTableSchema schema) { + public static List> get_indexed_sort_map(List> sort_map, IBlueTableSchema schema) + { List> rv = new List>(); foreach (Pair p in sort_map) rv.Add(new Pair(schema[p.first].colNo, p.second)); @@ -1510,13 +1755,15 @@ namespace BCG { /// empty strings etc are sent to the last public static int compare_send_dontcare_to_last(IBlueRow r1, string r1_col_name, IBlueRow r2, string r2_col_name, - bool low_to_high) { + bool low_to_high) + { object o1 = BlueDataUtility.get_row_value(r1, r1_col_name, DBNull.Value), o2 = BlueDataUtility.get_row_value(r2, r2_col_name, DBNull.Value); return compare_send_dontcare_to_last(o1, o2, low_to_high); } - public static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) { + public static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) + { if (is_dont_care_val(o1)) return is_dont_care_val(o2) ? 0 : 1; else @@ -1524,7 +1771,8 @@ namespace BCG { : ((IComparable)o2).CompareTo(o1)); } - public static bool is_dont_care_val(object o) { + public static bool is_dont_care_val(object o) + { return o == null || o == DBNull.Value || o is string && String.IsNullOrEmpty((string)o) || o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) || @@ -1536,22 +1784,26 @@ namespace BCG { #region Tree Aggregation - public static int[] convert_aggregation_tree(string[] aggregation_tree_col_names, IBlueTableSchema schema) { + public static int[] convert_aggregation_tree(string[] aggregation_tree_col_names, IBlueTableSchema schema) + { List rv = new List(); foreach (string col_name in aggregation_tree_col_names) rv.Add(schema[col_name].colNo); return rv.ToArray(); } - public static bool is_leaf_node(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) { + public static bool is_leaf_node(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) + { return BlueDataUtility.get_row_value(r, depth_in_tree_col_no, -1) == (aggregation_tree.Count() - 1); } - public static bool is_leaf_node(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) { + public static bool is_leaf_node(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) + { return BlueDataUtility.get_row_value(r, depth_in_tree_col_name, -1) == (aggregation_tree.Count() - 1); } - public static string generate_aggregation_based_id(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) { + public static string generate_aggregation_based_id(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) + { int aggregation_priority = BlueDataUtility.get_row_value(r, depth_in_tree_col_no, -1); string rv = String.Empty; for (int p = 0; p <= aggregation_priority; p++) @@ -1561,7 +1813,8 @@ namespace BCG { /// Generates a string based id for a row in a tree /// by appending the values of the nodes above it - public static string generate_aggregation_based_id(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) { + public static string generate_aggregation_based_id(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) + { int aggregation_priority = BlueDataUtility.get_row_value(r, depth_in_tree_col_name, -1); string rv = String.Empty; for (int p = 0; p <= aggregation_priority; p++) @@ -1573,12 +1826,15 @@ namespace BCG { /// Given an aggregation hierarchy tree, figures if the first /// arg is a parent of the second arg. /// - public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, string depth_in_tree_col_name, string[] aggregation_tree) { + public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, string depth_in_tree_col_name, string[] aggregation_tree) + { System.Diagnostics.Debug.Assert(parent_row.Contains(depth_in_tree_col_name) && child_row.Contains(depth_in_tree_col_name)); int parent_depth = BlueDataUtility.get_row_value(parent_row, depth_in_tree_col_name, Int32.MaxValue), child_depth = BlueDataUtility.get_row_value(child_row, depth_in_tree_col_name, Int32.MaxValue); - if (parent_depth < child_depth) { - for (int p = 0; p <= parent_depth; p++) { + if (parent_depth < child_depth) + { + for (int p = 0; p <= parent_depth; p++) + { string col_name = aggregation_tree.ElementAt(p); object parent_val = BlueDataUtility.get_row_value(parent_row, col_name, DBNull.Value), child_val = BlueDataUtility.get_row_value(child_row, col_name, DBNull.Value); @@ -1586,15 +1842,19 @@ namespace BCG { return false; } return true; - } else + } + else return false; } - public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, int depth_in_tree_col_no, int[] aggregation_tree) { + public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, int depth_in_tree_col_no, int[] aggregation_tree) + { int parent_depth = BlueDataUtility.get_row_value(parent_row, depth_in_tree_col_no, Int32.MaxValue), child_depth = BlueDataUtility.get_row_value(child_row, depth_in_tree_col_no, Int32.MaxValue); - if (parent_depth < child_depth) { - for (int p = 0; p <= parent_depth; p++) { + if (parent_depth < child_depth) + { + for (int p = 0; p <= parent_depth; p++) + { int col_no = aggregation_tree[p]; object parent_val = BlueDataUtility.get_row_value(parent_row, col_no, DBNull.Value), child_val = BlueDataUtility.get_row_value(child_row, col_no, DBNull.Value); @@ -1602,21 +1862,26 @@ namespace BCG { return false; } return true; - } else + } + else return false; } - public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, string depth_in_tree_col_name, string[] aggregation_tree, Dictionary sort_map) { + public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, string depth_in_tree_col_name, string[] aggregation_tree, Dictionary sort_map) + { if (BlueDataUtility.is_parent(r1, r2, depth_in_tree_col_name, aggregation_tree)) return -1; else if (BlueDataUtility.is_parent(r2, r1, depth_in_tree_col_name, aggregation_tree)) return 1; else if (get_row_value(r1, "id", Int32.MaxValue).Equals(get_row_value(r2, "id", Int32.MaxValue))) return 0; - else { - for (int p = 0; p < aggregation_tree.Count(); p++) { + else + { + for (int p = 0; p < aggregation_tree.Count(); p++) + { /// sort_map is used to leaf nodes - if (p == aggregation_tree.Count() - 1) { + if (p == aggregation_tree.Count() - 1) + { /// sorting for leaf nodes is special, since they can be /// sorted by data columns System.Diagnostics.Debug.Assert(BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree)); @@ -1636,17 +1901,21 @@ namespace BCG { } } - public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, int depth_in_tree_col_no, int[] aggregation_tree, Pair[] sort_map) { + public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, int depth_in_tree_col_no, int[] aggregation_tree, Pair[] sort_map) + { if (BlueDataUtility.is_parent(r1, r2, depth_in_tree_col_no, aggregation_tree)) return -1; else if (BlueDataUtility.is_parent(r2, r1, depth_in_tree_col_no, aggregation_tree)) return 1; else if (get_row_value(r1, "id", Int32.MaxValue).Equals(get_row_value(r2, "id", Int32.MaxValue))) return 0; - else { - for (int p = 0; p < aggregation_tree.Count(); p++) { + else + { + for (int p = 0; p < aggregation_tree.Count(); p++) + { /// sort_map is used to leaf nodes - if (p == aggregation_tree.Count() - 1) { + if (p == aggregation_tree.Count() - 1) + { /// sorting for leaf nodes is special, since they can be /// sorted by data columns //System.Diagnostics.Debug.Assert(BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree)); @@ -1670,123 +1939,169 @@ namespace BCG { #region Column accessors - public static T get_array_value(Array r, int col_no, T error_val) { - try { + public static T get_array_value(Array r, int col_no, T error_val) + { + try + { object rv = (r != null && col_no > -1 && col_no < r.Length) ? r.GetValue(col_no) : DBNull.Value; if (rv == null || rv == DBNull.Value) return error_val; else return (T)Convert.ChangeType(rv, typeof(T)); - } catch { + } + catch + { return error_val; } } - public static DateTime get_array_datetime_value(Array r, int col_no, DateTime error_val) { - try { + public static DateTime get_array_datetime_value(Array r, int col_no, DateTime error_val) + { + try + { object rv = (r != null && col_no > -1 && col_no < r.Length) ? r.GetValue(col_no) : error_val; return _get_datetime_value(rv, error_val); - } catch { + } + catch + { return error_val; } } - public static T get_row_value(IBlueRow r, int col_no, T error_val) { - try { + public static T get_row_value(IBlueRow r, int col_no, T error_val) + { + try + { object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : DBNull.Value; if (rv == null || rv == DBNull.Value) return error_val; else return (T)Convert.ChangeType(rv, typeof(T)); - } catch { + } + catch + { return error_val; } } - public static T get_row_value(IBlueRow r, string col_name, T error_val) { - try { + public static T get_row_value(IBlueRow r, string col_name, T error_val) + { + try + { object rv = (r != null && r.Contains(col_name)) ? r[col_name] : DBNull.Value; if (rv == null || rv == DBNull.Value) return error_val; else return (T)Convert.ChangeType(rv, typeof(T)); - } catch { + } + catch + { return error_val; } } - public static T get_row_enum_value(IBlueRow r, string col_name, T error_val) where T : struct { - try { + public static T get_row_enum_value(IBlueRow r, string col_name, T error_val) where T : struct + { + try + { object rv = r.Contains(col_name) ? r[col_name] : DBNull.Value; return _get_row_enum_value(rv, error_val); - } catch { + } + catch + { return error_val; } } - public static T get_row_enum_value(IBlueRow r, int col_no, T error_val) where T : struct { - try { + public static T get_row_enum_value(IBlueRow r, int col_no, T error_val) where T : struct + { + try + { object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : DBNull.Value; return _get_row_enum_value(rv, error_val); - } catch { + } + catch + { return error_val; } } - private static T _get_row_enum_value(object o, T error_val) where T : struct { - try { + private static T _get_row_enum_value(object o, T error_val) where T : struct + { + try + { if (o == null || o == DBNull.Value) return error_val; - else if (o is string) { + else if (o is string) + { T result = error_val; if (Enum.TryParse((string)o, true, out result)) return result; return error_val; - } else if (o is Int32 || o is Int16 || o is UInt16 || o is UInt32) + } + else if (o is Int32 || o is Int16 || o is UInt16 || o is UInt32) return (T)o; else if (o is T) return (T)o; else return error_val; - } catch { + } + catch + { return error_val; } } - public static DateTime get_row_datetime_value(IBlueRow r, string col_name, DateTime error_val) { - try { + public static DateTime get_row_datetime_value(IBlueRow r, string col_name, DateTime error_val) + { + try + { object rv = (r != null && r.Contains(col_name)) ? r[col_name] : error_val; return _get_datetime_value(rv, error_val); - } catch { + } + catch + { return error_val; } } - public static DateTime get_row_datetime_value(IBlueRow r, int col_no, DateTime error_val) { - try { + public static DateTime get_row_datetime_value(IBlueRow r, int col_no, DateTime error_val) + { + try + { object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : error_val; return _get_datetime_value(rv, error_val); - } catch { + } + catch + { return error_val; } } - private static DateTime _get_datetime_value(object v, DateTime error_val) { - try { + private static DateTime _get_datetime_value(object v, DateTime error_val) + { + try + { if (v == null || v == DBNull.Value) return error_val; - else { + else + { if (v is DateTime) return (DateTime)v; - else if (v is string) { + else if (v is string) + { DateTime rv_dt = error_val; DateTime.TryParse((string)v, out rv_dt); return rv_dt; - } else { + } + else + { double oa_date = (double)Convert.ChangeType(v, typeof(Double)); return DateTime.FromOADate(oa_date); } } - } catch { + } + catch + { return error_val; } } @@ -1797,7 +2112,8 @@ namespace BCG { /// /// Reads a schema file name, and return the constructed schema /// - public static IBlueTableSchema createSchemaFromCSVSchemaFile(string schemaFileName) { + public static IBlueTableSchema createSchemaFromCSVSchemaFile(string schemaFileName) + { StreamReader sr = new StreamReader(schemaFileName); IBlueTableSchema retVal = createSchemaFromCSVSchemaFile(sr); sr.Close(); @@ -1808,7 +2124,8 @@ namespace BCG { /// Builds a schema from a schema file. Note that the stream is not closed, so /// the calling code should do so /// - public static IBlueTableSchema createSchemaFromCSVSchemaFile(TextReader sr) { + public static IBlueTableSchema createSchemaFromCSVSchemaFile(TextReader sr) + { string tableName = sr.ReadLine(); ArrayList t = new ArrayList(); string l = ""; @@ -1816,12 +2133,14 @@ namespace BCG { t.Add(l.Trim()); string[] cols = t.ToArray(typeof(string)) as String[]; ColumnInfo[] cis = new ColumnInfo[cols.Length]; - for (int i = 0; i < cols.Length; i++) { + for (int i = 0; i < cols.Length; i++) + { string[] colTags = cols[i].Split('/'); cis[i] = new ColumnInfo(colTags[0], i, TypeInstanceLoader.loadType(colTags[1])); } IBlueTableSchema retVal = new BlueTableSchema(cis, tableName); - if (retVal["id"] == null) { + if (retVal["id"] == null) + { ArrayList temp = new ArrayList(retVal.columnInfo); temp.Add(new ColumnInfo("id", temp.Count, typeof(System.Int32))); retVal = new BlueTableSchema(retVal.tableName, @@ -1834,13 +2153,17 @@ namespace BCG { /// Creates an IBlueRow from a CSV data file row; if the string already contains an id, /// the id argument is ignored /// - public static IBlueRow createRowFromCSVFileRow(string fileRow, IBlueTableSchema schema, int id) { + public static IBlueRow createRowFromCSVFileRow(string fileRow, IBlueTableSchema schema, int id) + { string[] toks = CSVParser.parse(fileRow); ArrayList rowList = new ArrayList(); - if (toks.Length == schema.columnNames.Length) { + if (toks.Length == schema.columnNames.Length) + { for (int i = 0; i < toks.Length; i++) rowList.Add(myConvertTo(schema[i].colType, toks[i])); - } else { + } + else + { int idIndex = schema["id"].colNo; for (int i = 0; i < toks.Length; i++) rowList.Add(myConvertTo(schema[i < idIndex ? i : i + 1].colType, toks[i])); @@ -1852,7 +2175,8 @@ namespace BCG { /// /// Returns the CSV data string corresponding to the row /// - public static string toCSVDataString(IBlueRow r) { + public static string toCSVDataString(IBlueRow r) + { string retVal = ""; for (int i = 0; i < r.schema.columnNames.Length; i++) retVal += @@ -1862,7 +2186,8 @@ namespace BCG { return retVal.Remove(retVal.Length - 1, 1); } - public static string[] toCSVSchemaFileStrings(IBlueTableSchema schema) { + public static string[] toCSVSchemaFileStrings(IBlueTableSchema schema) + { string[] retVal = new string[2]; retVal[0] = schema.tableName; foreach (ColumnInfo ci in schema.columnInfo) @@ -1874,11 +2199,13 @@ namespace BCG { /// /// Converts a CSV data value to its original type. /// - private static object myConvertTo(Type t, string val) { + private static object myConvertTo(Type t, string val) + { val = val.Trim(); if (t.Equals(typeof(System.String))) return val; - else if (val != "") { + else if (val != "") + { if (t.Equals(typeof(System.Char))) return Convert.ToChar(val); if (t.Equals(typeof(System.Int16))) @@ -1897,17 +2224,21 @@ namespace BCG { return DBNull.Value; } - private static object myToDateTime(string val) { - try { + private static object myToDateTime(string val) + { + try + { return Convert.ToDateTime(val); - } catch (FormatException) //datetime is not stored in one of the string formats - { + } + catch (FormatException) //datetime is not stored in one of the string formats + { return val.IndexOf('.') > -1 ? DateTime.FromOADate(Convert.ToDouble(val)) : new DateTime(Convert.ToInt64(val)); } } - public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) { + public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) + { StreamReader sr = new StreamReader(fileAddress); string cols = sr.ReadLine(); string[] columns = cols.Split(','); @@ -1930,15 +2261,18 @@ namespace BCG { /// /// Uses a cached compiled expression to implement a truly generic add function. Will throw if given types that cannot be added. /// - internal class MagicAdder { + internal class MagicAdder + { static Func add; - static MagicAdder() { + static MagicAdder() + { ParameterExpression left = Expression.Parameter(typeof(T), "a"); ParameterExpression right = Expression.Parameter(typeof(T), "b"); add = (Func)Expression.Lambda(Expression.Add(left, right), left, right).Compile(); } - public static T Add(T a, T b) { + public static T Add(T a, T b) + { return add(a, b); } } @@ -1946,7 +2280,8 @@ namespace BCG { #region Misc - public static void swap(ref T t1, ref T t2) { + public static void swap(ref T t1, ref T t2) + { T temp = t1; t1 = t2; t2 = temp; diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs index 5d9f84537..5024be042 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs @@ -5,14 +5,12 @@ using System.Data; //using BCG.Utilities; using System.Linq; -namespace BCG -{ +namespace BCG { /// /// Anologue of the dotNet DataTable. /// [Serializable] - public class BlueTable : IBlueRowCollection - { + public class BlueTable : IBlueRowCollection { /// /// Hashtable indexed on "id" to speed up removes /// @@ -29,22 +27,19 @@ namespace BCG /// all the data over /// /// - public BlueTable(DataTable table) - { + public BlueTable(DataTable table) { ColumnInfo[] cis = new ColumnInfo[table.Columns.Count]; int i = 0; foreach (DataColumn col in table.Columns) cis[i] = new ColumnInfo(col.ColumnName, i++, col.DataType); mySchema = new BlueTableSchema(cis, table.TableName); - foreach (DataRow row in table.Rows) - { + foreach (DataRow row in table.Rows) { IBlueRow newRow = new BlueRow(mySchema, row.ItemArray); myIdIndex[newRow["id"]] = newRow; } } - public BlueTable(IBlueRowCollection table) - { + public BlueTable(IBlueRowCollection table) { mySchema = table.schema; foreach (IBlueRow row in table) addRow(row); @@ -52,12 +47,10 @@ namespace BCG public BlueTable(string tableName, IEnumerable objects, - params CMap[] columnMappers) - { + params CMap[] columnMappers) { // extract schema from mappers ColumnInfo[] cols = new ColumnInfo[columnMappers.Length]; - for (int i = 0; i < columnMappers.Length; i++) - { + for (int i = 0; i < columnMappers.Length; i++) { cols[i] = new ColumnInfo( columnMappers[i].getColumnName(), i, columnMappers[i].getColumnType()); @@ -66,8 +59,7 @@ namespace BCG // load rows int rowNumber = 0; - foreach (object rowObject in objects) - { + foreach (object rowObject in objects) { object[] values = new object[columnMappers.Length]; for (int cn = 0; cn < columnMappers.Length; cn++) values[cn] = columnMappers[cn].getColumnValue(rowNumber, rowObject); @@ -87,10 +79,8 @@ namespace BCG /// /// Returns the row by the index, interface property /// - public virtual IBlueRow this[int id] - { - get - { + public virtual IBlueRow this[int id] { + get { object row = myIdIndex[id]; if (row == null) throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName); @@ -105,8 +95,7 @@ namespace BCG public virtual bool containsID(int id) { return myIdIndex[id] != null; } - public virtual bool tryGetValue(int id, out IBlueRow row) - { + public virtual bool tryGetValue(int id, out IBlueRow row) { row = (IBlueRow)myIdIndex[id]; return (row != null); } @@ -114,8 +103,7 @@ namespace BCG #endregion #region BlueTable members - public virtual void clear() - { + public virtual void clear() { myIdIndex.Clear(); } @@ -123,8 +111,7 @@ namespace BCG /// If a row with the same id exists, it replaces it /// /// - public virtual void forceAddRow(IBlueRow r) - { + public virtual void forceAddRow(IBlueRow r) { myIdIndex[r["id"]] = r; } @@ -132,43 +119,41 @@ namespace BCG /// Adds a new row to the table. /// /// - public virtual void addRow(IBlueRow row) - { + public virtual void addRow(IBlueRow row) { object id = row["id"]; if (id == null || id == DBNull.Value) throw new BlueDataException("The ID of the row being added is null/DNBull"); - try - { + try { myIdIndex.Add(id, row); - } - catch (ArgumentException) - { - throw new BlueDataException($"ID {id} already exists in the table: {mySchema.tableName} , duplication of field ID not allowed"); + } catch (ArgumentException) { + throw new BlueDataException($"ID: {id} already exists in the table: {mySchema.tableName}"); } } - public virtual void addRow(DataRow row) - { + public virtual void addRow(DataRow row) { object val = row["id"]; if (val == null || val == DBNull.Value) throw new BlueDataException("The ID of the row being added is null/DNBull"); - if (myIdIndex[val] != null) - throw new BlueDataException("ID already exists in the table, duplication of field ID not allowed"); + if (myIdIndex[val] != null) { + string err_msg = $"ID: {val} already exists in the table: {mySchema.tableName} duplication of field ID not allowed"; + // temp TODOds remove it + bts.Log.assert(false, err_msg); + throw new BlueDataException(err_msg); + + } IBlueRow newRow = new BlueRow(mySchema, row); myIdIndex[val] = newRow; } - public virtual void addRows(IBlueRowCollection rows) - { + public virtual void addRows(IBlueRowCollection rows) { foreach (IBlueRow nr in rows) addRow(nr); } - public virtual void addRows(IEnumerable rows) - { + public virtual void addRows(IEnumerable rows) { foreach (IBlueRow nr in rows) addRow(nr); } @@ -185,37 +170,32 @@ namespace BCG /// Removes a row specified by the id. The index helps in making this operation O(1) /// /// - public virtual bool deleteRow(int id) - { + public virtual bool deleteRow(int id) { if (myIdIndex[id] == null) return false; myIdIndex.Remove(id); return true; } - public virtual bool deleteRows(IBlueRowCollection rows) - { + public virtual bool deleteRows(IBlueRowCollection rows) { bool retVal = true; foreach (IBlueRow r in rows) retVal &= deleteRow(r); return retVal; } - public virtual bool deleteRows(IEnumerable rows) - { + public virtual bool deleteRows(IEnumerable rows) { bool retVal = true; foreach (IBlueRow r in rows) retVal &= deleteRow(r); return retVal; } - public virtual void changeRow(IBlueRow r) - { + public virtual void changeRow(IBlueRow r) { myIdIndex[r["id"]] = r; } - public virtual void changeRows(IBlueRowCollection rows) - { + public virtual void changeRows(IBlueRowCollection rows) { foreach (IBlueRow r in rows) changeRow(r); } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator.cs index e2a5b7b97..4d55a1e28 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator.cs @@ -42,7 +42,7 @@ namespace BCG Delegate.Combine (myOnUpdateDelegate, value) as TableUpdateDelegate; if (myStateTable != null) value (myStateTable, myStateTable, new BlueTable (myStateTable.schema)); - } + } remove { // BlueEventLog.logEvent ("TableSubscriber being Unsubscribed for tag :" + myTag, BlueEventLevel.Information); @@ -64,6 +64,7 @@ namespace BCG public void update(Stream[] streams) { + bts.Log.Gen.info($"TableAccumulator.update tag: {myTag} n_streams: {streams.Length} stream[0].len: {streams[0].Length - streams[0].Position}"); //checking the condition as a subscriber might unsubscribe in a callback and there might be more than one //data element for that tag in the ViewStreamDatabase if (myOnUpdateDelegate != null) diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializationController.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializationController.cs index d27998c44..07a2e435e 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializationController.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializationController.cs @@ -3,16 +3,15 @@ using System.IO; using System.Collections; using BCG.Utilities; -namespace BCG -{ +namespace BCG { /// /// Controls (ie creates dynamically) an IClientDeserializer and then passes data to it. /// - public class ClientDeserializationController : IClientDeserializer - { + public class ClientDeserializationController : IClientDeserializer { private IClientDeserializer myDeserializer; - - public ClientDeserializationController () {} + private int _n_desers = 0; + + public ClientDeserializationController() { } #region IClientDeserializer Members @@ -23,42 +22,41 @@ namespace BCG /// /// /// - public void deserialize (Stream[] streams, out byte[] theOps, out Array theRows) - { - if (myDeserializer != null) - { - myDeserializer.deserialize (streams, out theOps, out theRows); - } - else - { - Stream totalStream = UtilityMethods.getAggregatedStream (streams); + public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) { + bts.Log.Gen.info($"ClientDeserializationController.deserialize len: {streams[0].Length - streams[0].Position}"); + if (myDeserializer != null) { + myDeserializer.deserialize(streams, out theOps, out theRows); + } else { + Stream totalStream = UtilityMethods.getAggregatedStream(streams); long initialPos = totalStream.Position; - BinaryReader br = new BinaryReader (totalStream); + BinaryReader br = new BinaryReader(totalStream); byte op; - try - { - op = br.ReadByte (); - } - catch (EndOfStreamException) - { + try { + op = br.ReadByte(); + } catch (EndOfStreamException) { op = Byte.MaxValue; } - if (op == StreamOpCode.create) - { - br.ReadString (); //pubsubclientdeserializer - string deserName = br.ReadString (); - myDeserializer = Activator.CreateInstance (Type.GetType (deserName)) + if (op == StreamOpCode.create) { + br.ReadString(); //pubsubclientdeserializer + string deserName = br.ReadString(); + myDeserializer = Activator.CreateInstance(Type.GetType(deserName)) as IClientDeserializer; totalStream.Position = initialPos; - myDeserializer.deserialize (new Stream[] {totalStream}, out theOps, out theRows); - } - else - { + myDeserializer.deserialize(new Stream[] { totalStream }, out theOps, out theRows); + } else { // BlueEventLog.logEvent ("Received a non create before a create", BlueEventLevel.Warning); - theOps = new byte[]{}; - theRows = new object[]{}; + theOps = new byte[] { }; + theRows = new object[] { }; } } + var msgs = StreamIOMessage.assemble_msgs(theOps, theRows); + int i = 0; + foreach (var m in msgs) { + bts.Log.Gen.info($"ClientDeserializationController n_desers: {_n_desers} msg no: {i++}: {m}"); + } + _n_desers++; + // log it here + } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalRowDeserializer.cs index 6ba3158e7..d696615ab 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalRowDeserializer.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalRowDeserializer.cs @@ -5,61 +5,61 @@ using System.IO; using BCG.Utilities; //using BCG.Utilities.BlueConfiguration; -namespace BCG -{ +namespace BCG { /// /// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process /// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations) /// - internal class ClientUniversalRowDeserializer : IClientRowDeserializer - { + internal class ClientUniversalRowDeserializer : IClientRowDeserializer { private IBlueTableSchema mySchema; private int myNullMapLength = 0; private PrimitiveDeserializationHelper myDeserializationHelper; private BitArray myNullMap; private int[] mySerIndexToSchemaIndexMap; - - public ClientUniversalRowDeserializer (IBlueTableSchema schema) - { - myDeserializationHelper = new PrimitiveDeserializationHelper (mySchema = schema); - myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8))); - myInitReorderSchema (); - + public ClientUniversalRowDeserializer(IBlueTableSchema schema) { + myDeserializationHelper = new PrimitiveDeserializationHelper(mySchema = schema); + myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length) / 8))); + myInitReorderSchema(); + + } #region Schema Reordering initialization - private void myInitReorderSchema () - { - int idColIndex = mySchema["id"].colNo; - int[] temp = new int [mySchema.columnInfo.Length]; - foreach (ColumnInfo ci in mySchema.columnInfo) - { - temp[ci.colNo] = - ci.colNo == idColIndex ? 0 : - ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo; + private void myInitReorderSchema() { + try { + int idColIndex = mySchema["id"].colNo; + int[] temp = new int[mySchema.columnInfo.Length]; + foreach (ColumnInfo ci in mySchema.columnInfo) { + temp[ci.colNo] = + ci.colNo == idColIndex ? 0 : + ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo; + } + mySerIndexToSchemaIndexMap = new int[temp.Length]; + for (int i = 0; i < temp.Length; i++) + mySerIndexToSchemaIndexMap[temp[i]] = i; + } catch (Exception ex) { + string schema_name = mySchema == null ? "NULL" : mySchema.tableName; + string err_msg = $"ClientUniversalRowDeserializer.myInitReorderSchema table: {schema_name} ex: {ex}"; + bts.Log.Gen.error(err_msg); + throw new ApplicationException(err_msg); } - mySerIndexToSchemaIndexMap = new int[temp.Length]; - for (int i = 0; i < temp.Length; i ++) - mySerIndexToSchemaIndexMap [temp[i]] = i; } #endregion #region IClientRowDeserializer Members - public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) - { - br.ReadInt32 (); //length information for the row not needed - myNullMap = new BitArray(br.ReadBytes (myNullMapLength)); - object[] values = new object [mySchema.columnNames.Length]; - for (int i = 0; i < values.Length; i++) - { + public void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) { + br.ReadInt32(); //length information for the row not needed + myNullMap = new BitArray(br.ReadBytes(myNullMapLength)); + object[] values = new object[mySchema.columnNames.Length]; + for (int i = 0; i < values.Length; i++) { int colNo = mySerIndexToSchemaIndexMap[i]; - values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn (colNo, br); + values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn(colNo, br); } - r = new BlueRow(mySchema, values); + r = new BlueRow(mySchema, values); } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientDeserializer.cs index 6be1a959a..a4e4c030d 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientDeserializer.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientDeserializer.cs @@ -1,22 +1,149 @@ using System; -using System.Collections; +using System.Collections.Generic; using System.IO; +using System.Linq; + +namespace BCG { + public class StreamIOMessage { + public enum StreamOp { + Create = 0, + Add = 1, //adds a single row + Remove = 2, //removes a single row + Change = 3, //changes a single row + MultipleRemoveAdd = 4,//removes and adds multiple rows + MultipleChange = 5 //changes multiples rows + } + public class CreateMessage { + public IBlueRowCollection rows; + public override string ToString() { + string rv = "Create message "; + if (rows == null) { + rv += " rows: null"; + return rv; + } + foreach (IBlueRow r in rows) { + rv += $"id: {r["id"]} "; + } + return rv; + } + } + public class AddMessage { + public IBlueRow added_row; + } + public class RemoveMessage { + public IBlueRow removed_row; + } + public class ChangeMessage { + public IBlueRow changed_row; + } + public class MultipleRemoveAddMessage { + public IBlueRowCollection removed; + public IBlueRowCollection added; + public override string ToString() { + string rv = "MultipleRemoveAdd removed: "; + foreach (IBlueRow r in removed) { + rv += $"id: {r["id"]} "; + } + rv += "added: "; + foreach (IBlueRow r in added) { + rv += $"id: {r["id"]} "; + } + return rv; + } + } + public class MultipleChangeMessage { + public IBlueRowCollection changed; + } + + public StreamOp op; + public CreateMessage create_message = null; + public AddMessage add_message = null; + public RemoveMessage remove_message = null; + public ChangeMessage change_message = null; + public MultipleRemoveAddMessage multiple_remove_add_message = null; + public MultipleChangeMessage multiple_change_message = null; + public StreamIOMessage(byte op_, object data) { + op = (StreamIOMessage.StreamOp)(op_); + switch (op) { + case StreamIOMessage.StreamOp.Create: + IBlueRowCollection state = null; + // TODO, not sure why it can be an array and by itself + if (data is IBlueRowCollection) state = data as IBlueRowCollection; + else if (data is IBlueRowCollection[]) state = (data as IBlueRowCollection[]).ElementAt(0); + create_message = new CreateMessage() { rows = state }; + break; + case StreamIOMessage.StreamOp.Add: + bts.Log.assert(false); + break; + case StreamIOMessage.StreamOp.Remove: + bts.Log.assert(false); + break; + case StreamIOMessage.StreamOp.Change: + bts.Log.assert(false); + break; + case StreamIOMessage.StreamOp.MultipleRemoveAdd: + IBlueRowCollection[] row_args = data as IBlueRowCollection[]; + multiple_remove_add_message = new MultipleRemoveAddMessage() { removed = row_args[0], added = row_args[1] }; + break; + case StreamIOMessage.StreamOp.MultipleChange: + //IBlueRowCollection[] row_args = data as IBlueRowCollection[]; + multiple_change_message = new MultipleChangeMessage() { changed = (data as IBlueRowCollection[])[0] }; + break; + } + } + + + public override string ToString() { + switch (op) { + case StreamIOMessage.StreamOp.Create: + return create_message.ToString(); + break; + case StreamIOMessage.StreamOp.Add: + bts.Log.assert(false); + return String.Empty; + break; + case StreamIOMessage.StreamOp.Remove: + bts.Log.assert(false); + return String.Empty; + break; + case StreamIOMessage.StreamOp.Change: + bts.Log.assert(false); + return String.Empty; + break; + case StreamIOMessage.StreamOp.MultipleRemoveAdd: + return multiple_remove_add_message.ToString(); + break; + case StreamIOMessage.StreamOp.MultipleChange: + return multiple_change_message.ToString(); + break; + } + bts.Log.assert(false); + return String.Empty; + } + public static IEnumerable assemble_msgs(byte[] ops, Array op_data) { + int i = 0; + List rv = new List(); + foreach (byte op in ops) { + object data = op_data.GetValue(i++); + rv.Add(new StreamIOMessage(op, data)); + } + return rv; + } + } -namespace BCG -{ /// /// Performs deserialization inside the client app. The client gets either the complete data set or the /// increments from PubSub and this interface makes sense out of them from a tabular/datastream stand point. /// - public interface IClientDeserializer - { + public interface IClientDeserializer { /// /// deserializes the data received by the client from the pubsub /// /// the data in byte format received from pubsub /// the operations that the data maps onto /// the data corresponding to the operations, is an array of IblueRowCollection[] - void deserialize (Stream[] streams, out byte[] theOps, out Array theRows); + void deserialize(Stream[] streams, out byte[] theOps, out Array theRows); } - } + + diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientRowDeserializer.cs index 5f35a2423..8a2e22e8d 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientRowDeserializer.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientRowDeserializer.cs @@ -2,13 +2,11 @@ using System; using System.IO; using System.Collections; -namespace BCG -{ +namespace BCG { /// /// Deserializes one row. the current state is also provided as some deserialization schemes might use it. /// - public interface IClientRowDeserializer - { - void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r); + public interface IClientRowDeserializer { + void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r); } } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTablePublisher.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTablePublisher.cs index 51acf09e8..8debb5c3d 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTablePublisher.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTablePublisher.cs @@ -16,7 +16,7 @@ namespace BCG { public PubTablePublisher(string tag, PublisherCore publisherCore) { _tag = tag; - bts.Log.Gen.info($"PubTablePublisher created for tag: {tag}"); + bts.Log.Gen.info($"PubTablePublisher.cons created for tag: {tag}"); myPubCore = publisherCore; mySerializer = new BlueDataSerializationController(); } @@ -54,10 +54,14 @@ namespace BCG { out data); bts.Log.Gen.info($"PubTablePublisher.createTable tag: {_tag} rows: {rows.Count} data: {data.Length}"); myPublish(data, rows.schema.tableName); + // byte[] ops; + // Array rows_ds; + // _test_deserializer.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows_ds); } + //private ClientDeserializationController _test_deserializer = new ClientDeserializationController(); public virtual void batchRemoveAdd(IBlueRowCollection delRows, IBlueRowCollection addRows) { - byte[] data; + byte[] data = null; if (myIsRealChange(delRows, addRows)) batchChange(addRows); else { @@ -66,6 +70,9 @@ namespace BCG { out data); myPublish(data, delRows.schema.tableName); } + // byte[] ops; + // Array rows_ds; + // _test_deserializer.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows_ds); } public virtual void batchChange(IBlueRowCollection rows) { diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs index 1775b0fa2..16a58ff6e 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Program.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs @@ -1,40 +1,84 @@ using System; +using mbts.config; +using bts.utils; using System.Threading.Tasks; +using Bts.DataStreamExpressions; +using System.Collections.Generic; using BCG; -bts.utils.BTSNativeMethods.provide_command_line(args, args.Length); -mbts.config.Configuration.initialize(); bts.Log.Gen.info($"Starting DSPublisher"); Console.WriteLine("Starting DSPublisher..."); -// static async Task _run() { -// var login = "bts"; -// var passwd = String.Empty; -// Console.WriteLine("Starting DSQA... "); -// // Log.notnull(login); -// // Log.notnull(passwd); -// // var service_ctx = bts.api.core.ServiceContext.create(); -// await service_ctx.login(login, passwd); -// } -//Task.Run(async () => await _run()); - -_publish_stream(); - -void _publish_stream() { - - BCG.ViewStream vs = new LiteralStream("x", "420.0");// new BBOStream(); - //new BCG.MDStream(); - ViewPublisher vpub = new ViewPublisher(String.Empty, new PubTableBroker(), vs); - vs.onChange += on_change; +static void init(string[] args) +{ + Console.WriteLine("Hit key to continue"); + Console.ReadLine(); + Func login = async (l, p) => + { + var login = l; + var passwd = p; + var service_ctx = bts.api.core.ServiceContext.create(); + await service_ctx.login(login, passwd); + }; + BTSNativeMethods.provide_command_line(args, args.Length); + Configuration.initialize(); + Task.Run(async () => await login("bts", String.Empty)); } -while (true) { - System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1)); - ViewStreamDatabase.instance.doWork(); + +init(args); +_publish_streams(); + +static ViewStream build_stream(string stream_expression) +{ + try + { + var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); + var rv = exp.getView(new Dictionary(0)); + return rv; + + } + catch (Exception ex) + { + bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}"); + return null; + } } -void on_change(object obj) { - DataPacket dp = obj as DataPacket; - dp.newRows.print_table((col) => { return 10; }); +void _publish_streams() +{ + PubTableBroker pb_broker = new PubTableBroker(); + CloudStreamManager csm = new CloudStreamManager(); + // get it from config + Dictionary streams_to_publish = new Dictionary() { + {"bbo","bbo()"}, + {"instruments","instruments()"}, + //{"timesales","timesales(10000)"}, + {"books","books()"}, + // {"seconds_1_10","second(1,10)"}, + {"literal_420","literal(\"x\", \"420.0\")"}, + // {"bbo_ts","timeSeries(bbo())"} + }; + foreach (var kv in streams_to_publish) + { + var name = kv.Key; + var exp = kv.Value; + BCG.ViewStream vs = build_stream(exp); + ViewPublisher vpub = new ViewPublisher(name, pb_broker, vs); + vs.onChange += on_change; + } + while (true) + { + System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1)); + ViewStreamDatabase.instance.doWork(); + csm.doWork(); + } +} + + + +void on_change(object obj) +{ + bts.Log.Gen.info($"DSPublisher on_change dp: {obj}"); } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs index bb15b444e..2fc290560 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs @@ -25,17 +25,19 @@ using System.Threading.Tasks; -namespace BCG { - public partial class MainForm : System.Windows.Forms.Form { - static async Task _run() { - var login = "bts"; - var passwd = String.Empty; - Console.WriteLine("Starting DSQA... "); - // Log.notnull(login); - // Log.notnull(passwd); - var service_ctx = bts.api.core.ServiceContext.create(); - await service_ctx.login(login, passwd); - } +namespace BCG +{ + public partial class MainForm : System.Windows.Forms.Form + { + // static async Task _run() { + // var login = "bts"; + // var passwd = String.Empty; + // Console.WriteLine("Starting DSQA... "); + // // Log.notnull(login); + // // Log.notnull(passwd); + // var service_ctx = bts.api.core.ServiceContext.create(); + // await service_ctx.login(login, passwd); + // } private bool myChangeStyle = false; private PubSubRoute[] myRoutes = PubSub3Config.instance.getSubRouting(); private PersistWindowState myWindowState; @@ -50,8 +52,9 @@ namespace BCG { private string myKQ; private TextWriter myConsoleOutput; - public MainForm() { - Task.Run(async () => await _run()); + public MainForm() + { + //Task.Run(async () => await _run()); InitializeComponent(); // myKQ = ConfigurationRequester.getApplicationQualifier(); myKQ = "TODO"; @@ -65,17 +68,20 @@ namespace BCG { this.Closing += new CancelEventHandler(myClosing); } - public string query_text { + public string query_text + { get { return myQueryTextBox.Text; } set { myQueryTextBox.Text = value; } } - private ViewStream myGetAllTagsStream() { + private ViewStream myGetAllTagsStream() + { // TODO: Port PublishedTagsStream from edge (then we can uncomment impl below) return new FilterStream(new PubSubServerStateStream(), "tag~='DS_'"); //return new LiteralStream("tag/String", "DS_Comedians;DS_Baldies"); } - private void mySetText() { + private void mySetText() + { this.Text = "Data Stream Query Analyzer | Site: " + mySite + (string.IsNullOrEmpty(myKQ) ? string.Empty : " | Key Qualifier: " + myKQ); myToolStripStatusLabelTime.Text = "Ready"; myToolStripStatusLabelRows.Text = string.Empty; @@ -85,10 +91,12 @@ namespace BCG { /// Gets the data in the form of a multilevel IDictionary /// /// - private IDictionary myGetLibraryData() { + private IDictionary myGetLibraryData() + { SortedList retVal = new SortedList(); string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames(); - foreach (string pName in primitiveNames) { + foreach (string pName in primitiveNames) + { string aName = PrimitiveManager.instance.getAssemblyName(pName); ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName); if (!retVal.Contains(aName)) @@ -103,12 +111,15 @@ namespace BCG { } - private void myLoadAssemblies() { + private void myLoadAssemblies() + { myTreeView.Nodes.Clear(); IDictionary libraryData = myGetLibraryData(); - foreach (DictionaryEntry de in libraryData) { + foreach (DictionaryEntry de in libraryData) + { TreeNode assemblyNode = new TreeNode(de.Key as String); - foreach (DictionaryEntry innerDe in (de.Value as IDictionary)) { + foreach (DictionaryEntry innerDe in (de.Value as IDictionary)) + { TreeNode primitiveNode = new TreeNode(innerDe.Key as string); foreach (string con in (innerDe.Value as ICollection)) primitiveNode.Nodes.Add(con); @@ -128,23 +139,27 @@ namespace BCG { }*/ } - private void myHandleTableListChange(object o) { + private void myHandleTableListChange(object o) + { IBlueRowCollection rows = (o as DataPacket).newRows; myUpdateTableList(); } #region Persistence Keys - public int textBoxHeight { + public int textBoxHeight + { get { return myQueryTextBox.Height; } set { myQueryTextBox.Height = value; } } #endregion - private void myTryExpression(string expStr) { + private void myTryExpression(string expStr) + { myFreezeView(false); // force unfreeze the view DataStreamExpression oldExp = myExp; - try { + try + { var exp = DataStreamExpressionsInterpreter.Evaluate(expStr); if (exp == null) @@ -171,7 +186,9 @@ namespace BCG { oldExp.getView(new Hashtable()).onChange -= new DataStream.ChangeDelegate(myCallback); */ //myQueryTextBox.Text = expStr; - } catch (Exception exc) { + } + catch (Exception exc) + { Log.Ui.warn(exc.ToString()); myDataGrid.DataSource = null; @@ -185,7 +202,8 @@ namespace BCG { } } - private string myExtractFromClass(TextReader r) { + private string myExtractFromClass(TextReader r) + { string line = r.ReadLine(); while (line != null && line.IndexOf("*Expression*") == -1) line = r.ReadLine(); @@ -198,7 +216,8 @@ namespace BCG { string result = ""; line = r.ReadLine(); - while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1) { + while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1) + { if (line != "") result += line.Substring(" ".Length) + "\n"; line = r.ReadLine(); @@ -209,17 +228,24 @@ namespace BCG { return result; } - public void loadFile(string fileName) { - try { + public void loadFile(string fileName) + { + try + { TextReader r = new StreamReader(fileName); - try { + try + { myQueryTextBox.Text = myExtractFromClass(r); - } finally { + } + finally + { r.Close(); } myLastFileName = fileName; this.Text = fileName; - } catch (Exception exc) { + } + catch (Exception exc) + { Log.Ui.warn(exc.ToString()); myDataGrid.DataSource = null; @@ -230,7 +256,8 @@ namespace BCG { } - private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code) { + private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code) + { myCSharpText.Text = "using System;\n\nnamespace BCG\n{"; @@ -250,7 +277,8 @@ namespace BCG { string lastVar = null; string variableDeclarations = ""; - foreach (string var in vars) { + foreach (string var in vars) + { variableDeclarations += " ViewStream " + var + ";\n"; lastVar = var; } @@ -280,13 +308,16 @@ namespace BCG { myCSharpText.Text += "}"; } - static DataGridViewColumn _column_factory(ColumnInfo ci) { - return new DataGridViewTextBoxColumn { + static DataGridViewColumn _column_factory(ColumnInfo ci) + { + return new DataGridViewTextBoxColumn + { HeaderText = ci.colName, Width = get_col_width(ci) }; - static int get_col_width(ColumnInfo ci) { + static int get_col_width(ColumnInfo ci) + { const int CHAR_WIDTH = 10; const int MIN_WIDTH = 50; const int MIN_DATE_WIDTH = 200; @@ -297,12 +328,14 @@ namespace BCG { } private HighPerformanceTimer _timer = new HighPerformanceTimer(); // this is the stream callback - private void myCallback(object o) { + private void myCallback(object o) + { _timer.start(); DataPacket dp = o as DataPacket; myCrono.stop(); Log.Ui.diag($"Received callback for {dp.schema.tableName}"); - if (o == null) { + if (o == null) + { System.Windows.Forms.MessageBox.Show(this, "Received a Null Data packet", "Invalid Data", MessageBoxButtons.OK, MessageBoxIcon.Error); return; @@ -319,7 +352,8 @@ namespace BCG { myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows); var force = false; - if (myChangeStyle && force) { + if (myChangeStyle && force) + { myChangeStyle = false; myDataGrid.Columns.Clear(); myDataGrid.Columns.AddRange(Array.ConvertAll(dp.schema.columnInfo, _column_factory)); @@ -346,11 +380,15 @@ namespace BCG { private DataStreamExpression myExp; - private void myTimer_Tick(object sender, System.EventArgs e) { - try { + private void myTimer_Tick(object sender, System.EventArgs e) + { + try + { ViewStreamDatabase.instance.doWork(); MasterWorker.getInstance(false).doWork(); - } catch (Exception exc) { + } + catch (Exception exc) + { Log.Ui.warn(exc.ToString()); myDataGrid.DataSource = null; @@ -361,22 +399,27 @@ namespace BCG { } } - class StringWrapper { - public StringWrapper(string s) { + class StringWrapper + { + public StringWrapper(string s) + { myTableName = s; } - public string tableName { + public string tableName + { get { return myTableName; } } private string myTableName; } - private void MainForm_Closed(object sender, System.EventArgs e) { + private void MainForm_Closed(object sender, System.EventArgs e) + { if (myExp != null) myExp.getView(new Dictionary(0)).onChange -= new DataStream.ChangeDelegate(myCallback); } - private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) { + private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) + { myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length; myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length; myQueryTextBox.SelectionColor = Color.Black; @@ -384,22 +427,24 @@ namespace BCG { myTryExpression(myQueryTextBox.Text); } - private void myToolStripMenuItemExecSel_Click(object sender, EventArgs e) { - int selstart = myQueryTextBox.SelectionStart; - int sellength = myQueryTextBox.SelectionLength; - myQueryTextBox.SelectionStart = 0; - myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length; - myQueryTextBox.SelectionColor = Color.Black; - myQueryTextBox.SelectionStart = selstart; - myQueryTextBox.SelectionLength = sellength; - myQueryTextBox.SelectionColor = Color.Crimson; - myTryExpression(myQueryTextBox.SelectedText); - myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length; - myQueryTextBox.SelectionLength = 0; - myQueryTextBox.SelectionColor = Color.Black; + private void myToolStripMenuItemExecSel_Click(object sender, EventArgs e) + { + // int selstart = myQueryTextBox.SelectionStart; + // int sellength = myQueryTextBox.SelectionLength; + // myQueryTextBox.SelectionStart = 0; + // myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length; + // myQueryTextBox.SelectionColor = Color.White; + // myQueryTextBox.SelectionStart = selstart; + // myQueryTextBox.SelectionLength = sellength; + // myQueryTextBox.SelectionColor = Color.Crimson; + // myTryExpression(myQueryTextBox.SelectedText); + // myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length; + // myQueryTextBox.SelectionLength = 0; + // myQueryTextBox.SelectionColor = Color.White; } - private void myMenuItemOpenClick(object sender, System.EventArgs e) { + private void myMenuItemOpenClick(object sender, System.EventArgs e) + { OpenFileDialog od = new OpenFileDialog(); od.Filter = "C# class (*.cs)|*.cs"; @@ -407,21 +452,24 @@ namespace BCG { loadFile(od.FileName); } - private void mySave(string fileName) { + private void mySave(string fileName) + { using (TextWriter w = new StreamWriter(fileName)) w.Write(myCSharpText.Text); myLastFileName = fileName; this.Text = fileName; } - private void myMenuItemSaveClick(object sender, System.EventArgs e) { + private void myMenuItemSaveClick(object sender, System.EventArgs e) + { if (myLastFileName == null) myMenuItemSaveAsClick(sender, e); else mySave(myLastFileName); } - private void myMenuItemSaveAsClick(object sender, System.EventArgs e) { + private void myMenuItemSaveAsClick(object sender, System.EventArgs e) + { SaveFileDialog sd = new SaveFileDialog(); if (myLastFileName != null) sd.FileName = myLastFileName; @@ -432,47 +480,56 @@ namespace BCG { mySave(sd.FileName); } - private void myTableListBox_DoubleClick(object sender, System.EventArgs e) { - if (myTableListBox.SelectedItem != null) { + private void myTableListBox_DoubleClick(object sender, System.EventArgs e) + { + if (myTableListBox.SelectedItem != null) + { string tableName = myTableListBox.SelectedItem.ToString(); myQueryTextBox.Text = myQueryTextBox.Text == "" ? tableName : myQueryTextBox.Text + "\n" + tableName; myTryExpression(myQueryTextBox.Text); } } - private void myDataGrid_MouseEnter(object sender, System.EventArgs e) { + private void myDataGrid_MouseEnter(object sender, System.EventArgs e) + { //myDataGrid.TableStyles[0].GridLineColor = Color.LightSteelBlue; // if (myDataGrid.TableStyles.Count > 0) // myDataGrid.TableStyles[0].HeaderBackColor = Color.LightSteelBlue; } - private void myDataGrid_MouseLeave(object sender, System.EventArgs e) { + private void myDataGrid_MouseLeave(object sender, System.EventArgs e) + { //myDataGrid.TableStyles[0].GridLineColor = Color.Gainsboro; // if (myDataGrid.TableStyles.Count > 0) // myDataGrid.TableStyles[0].HeaderBackColor = Color.Gainsboro; } - private void myCSharpText_MouseEnter(object sender, System.EventArgs e) { + private void myCSharpText_MouseEnter(object sender, System.EventArgs e) + { //myCSharpText.BackColor = Color.Gainsboro; myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Bold); } - private void myCSharpText_MouseLeave(object sender, System.EventArgs e) { + private void myCSharpText_MouseLeave(object sender, System.EventArgs e) + { //myCSharpText.BackColor = Color.White; myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Regular); } - private void myTableListBox_MouseEnter(object sender, System.EventArgs e) { + private void myTableListBox_MouseEnter(object sender, System.EventArgs e) + { //myTableListBox.BackColor = Color.Gainsboro; myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Bold); } - private void myTableListBox_MouseLeave(object sender, System.EventArgs e) { + private void myTableListBox_MouseLeave(object sender, System.EventArgs e) + { //myTableListBox.BackColor = Color.White; myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Regular); } - private void myLoad(object sender, EventArgs e) { + private void myLoad(object sender, EventArgs e) + { myFreezeView(false); myToolStripMenuItemSite.DropDownItems.Clear(); @@ -496,7 +553,9 @@ namespace BCG { { var ex = new Win32Exception(); // same as new Win32Exception(Marshal.GetLastWin32Error()) Console.WriteLine(ex.Message); - } else { + } + else + { Console.WriteLine("Set stdout worked."); } @@ -504,7 +563,9 @@ namespace BCG { { var ex = new Win32Exception(); // same as new Win32Exception(Marshal.GetLastWin32Error()) Console.WriteLine(ex.Message); - } else { + } + else + { Console.WriteLine("Set stderr worked."); } Console.WriteLine("Redirecting console ouput to this window..."); @@ -520,10 +581,12 @@ namespace BCG { //_library = new LibraryForm(this); } - private void myMenuItemSiteClick(object sender, System.EventArgs e) { + private void myMenuItemSiteClick(object sender, System.EventArgs e) + { ToolStripMenuItem mi = (ToolStripMenuItem)sender; - if (mi.Text != mySite) { + if (mi.Text != mySite) + { mySite = mi.Text; foreach (var m in mySiteItems) m.Checked = false; @@ -544,63 +607,78 @@ namespace BCG { } } - private void myClosing(object sender, CancelEventArgs e) { + private void myClosing(object sender, CancelEventArgs e) + { } - private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e) { - if (myTreeView.SelectedNode != null) { - if (myTreeView.SelectedNode.LastNode == null) { + private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e) + { + if (myTreeView.SelectedNode != null) + { + if (myTreeView.SelectedNode.LastNode == null) + { string sn = myTreeView.SelectedNode.Parent.Text + "()"; myQueryTextBox.Text = myQueryTextBox.Text == "" ? sn : myQueryTextBox.Text + "\n" + sn; } } } - private void myToolStripMenuItemExit_Click(object sender, EventArgs e) { + private void myToolStripMenuItemExit_Click(object sender, EventArgs e) + { Close(); } - private void myToolStripMenuItemFreeze_Click(object sender, EventArgs e) { + private void myToolStripMenuItemFreeze_Click(object sender, EventArgs e) + { myFreezeView(myTimer.Enabled); } - private void myFreezeView(bool freeze) { + private void myFreezeView(bool freeze) + { myTimer.Enabled = !freeze; myToolStripMenuItemFreeze.Checked = freeze; myToolStripMenuItemStep.Enabled = freeze; } - private void myToolStripMenuItemStep_Click(object sender, EventArgs e) { + private void myToolStripMenuItemStep_Click(object sender, EventArgs e) + { ViewStreamDatabase.instance.doWork(); } - private void myToolStripMenuItemLoadAss_Click(object sender, EventArgs e) { + private void myToolStripMenuItemLoadAss_Click(object sender, EventArgs e) + { OpenFileDialog ofd = new OpenFileDialog(); ofd.ShowReadOnly = true; DialogResult dr = ofd.ShowDialog(this); - if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK) { + if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK) + { PrimitiveManager.instance.loadAssembly(ofd.FileName); myLoadAssemblies(); } } - private void myToolStripMenuItemTest_Click(object sender, EventArgs e) { - if (myCurrentStream != null) { + private void myToolStripMenuItemTest_Click(object sender, EventArgs e) + { + if (myCurrentStream != null) + { myCurrentStream.onChange -= new DataStream.ChangeDelegate(myCallback); myCurrentStream.onChange += new DataStream.ChangeDelegate(myCallback); } } - private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e) { + private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e) + { myConsoleRichTextBox.Clear(); Console.WriteLine("Redirecting console ouput to this window..."); } - private void myTableFilterBox_TextChanged(object sender, EventArgs e) { + private void myTableFilterBox_TextChanged(object sender, EventArgs e) + { myUpdateTableList(); } - private void myUpdateTableList() { + private void myUpdateTableList() + { IBlueRowCollection rows = myAllTagsStream.rows; if (rows == null) return; @@ -615,12 +693,15 @@ namespace BCG { myTableListBox.Items.Clear(); SortedList sl = new SortedList(); if (string.IsNullOrEmpty(myTableFilterBox.Text)) - foreach (IBlueRow r in rows) { + foreach (IBlueRow r in rows) + { sl[r["tag"].ToString().Substring(PubStreamPrefixer.prefix("").Length)] = true; } - else { + else + { Regex tableRegex = new Regex(myTableFilterBox.Text); - foreach (IBlueRow r in rows) { + foreach (IBlueRow r in rows) + { string thisTable = r["tag"].ToString().Substring(PubStreamPrefixer.prefix("").Length); if (tableRegex.IsMatch(thisTable)) sl[thisTable] = true; @@ -638,7 +719,8 @@ namespace BCG { myTableListBox.EndUpdate(); } - private void testGridToolStripMenuItem_Click(object sender, EventArgs e) { + private void testGridToolStripMenuItem_Click(object sender, EventArgs e) + { HighPerformanceTimer timer = new HighPerformanceTimer(); //while (true) //{ @@ -660,9 +742,11 @@ namespace BCG { foreach (IBlueRow r in s.rows) rows.Add(r); - for (int i = 0; i < 100; i++) { + for (int i = 0; i < 100; i++) + { timer.start(); - rows.Sort(delegate (IBlueRow r1, IBlueRow r2) { + rows.Sort(delegate (IBlueRow r1, IBlueRow r2) + { return _sort_dicts_index(r1, r2, 5, true); }); timer.stop(); @@ -670,19 +754,22 @@ namespace BCG { } } - private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high) { + private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high) + { if (r1.Contains(column) && r2.Contains(column)) return compare_send_dontcare_to_last(r1[column], r2[column], low_to_high); else return 0; } - private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high) { + private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high) + { //return compare_send_dontcare_to_last(r1[col_no], r2[col_no], low_to_high); return ((IComparable)r1[col_no]).CompareTo(r2[col_no]); } - internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) { + internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) + { if (myIsDontCare(o1)) return myIsDontCare(o2) ? 0 : 1; else @@ -690,14 +777,16 @@ namespace BCG { : ((IComparable)o2).CompareTo(o1)); } - private static bool myIsDontCare(object o) { + private static bool myIsDontCare(object o) + { return o == null || o == DBNull.Value || o is string && String.IsNullOrEmpty((string)o) || o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) || o is float && (Single.IsInfinity((float)o) || Single.IsNaN((float)o)); } - private void _new_grid_form(object args) { + private void _new_grid_form(object args) + { //System.Timers.Timer t = new System.Timers.Timer(); ////t.SynchronizingObject //BCG.Utilities.BlueEventLog.logEvent(String.Format("New BTSTabularGridTestForm created on thread {0}", System.Threading.Thread.CurrentThread.ManagedThreadId), @@ -709,11 +798,13 @@ namespace BCG { private void libraryToolStripMenuItem_Click(object sender, EventArgs e) { } - private void myTableFilterLabel_Click(object sender, EventArgs e) { + private void myTableFilterLabel_Click(object sender, EventArgs e) + { } - private void MainForm_Load(object sender, EventArgs e) { + private void MainForm_Load(object sender, EventArgs e) + { } } diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs index 98da544f3..41d7e9590 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs @@ -5,16 +5,31 @@ using bts.utils; using mbts.config; using System; +using System.Linq; using System.Windows.Forms; +using System.Threading.Tasks; +using System.ComponentModel; static class Program { - [STAThread] - static void Main(string[] args) { + static void init(string[] args) { + Func login = async (l, p) => { + var login = l; + var passwd = p; + var service_ctx = bts.api.core.ServiceContext.create(); + await service_ctx.login(login, passwd); + }; BTSNativeMethods.provide_command_line(args, args.Length); Configuration.initialize(); - bts.Log.Gen.info($"Starting DSQA"); - Console.WriteLine("Starting DataStreamQueryAnalyzer..."); - Console.WriteLine("Starting Test Publisher"); + Task.Run(async () => await login("bts", String.Empty)); + } + [STAThread] + static void Main(string[] args) { + init(args); + string args_concat = String.Empty; + args.ToList().ForEach(t => args_concat += t); + var init_log = $"Starting DSQA args: {args_concat} len: {args.Length}"; + bts.Log.Gen.info(init_log); + Console.WriteLine(init_log); Console.WriteLine("Launching Main Window"); MainForm mainForm = new MainForm(); if (args.Length > 0) { diff --git a/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs b/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs index 32d5ebc96..68306d289 100644 --- a/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs +++ b/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs @@ -9,8 +9,7 @@ using System.IO; using System.Threading; using mbts; -namespace BCG -{ +namespace BCG { /// /// Keeps an ITCPConnection for talking to the server. It also monitors heartbeats /// from the server, and in case of missing hbs, informs its clients, and tries to @@ -23,8 +22,7 @@ namespace BCG /// public class PubSubServerManagedTCPConnection : IPubSubServerConnection, ITCPMessageHandler, - ITCPConnectionHandler - { + ITCPConnectionHandler { private ITCPConnection _connection, _new_connection; private readonly BlutoSubAddress _server_address; // if a message(with confirm) can not be sent , its id is queued here, @@ -43,98 +41,75 @@ namespace BCG public PubSubServerManagedTCPConnection(string serverURL) : this(new BlutoSubAddress(serverURL)) { } - public PubSubServerManagedTCPConnection(BlutoSubAddress server_address) - { + public PubSubServerManagedTCPConnection(BlutoSubAddress server_address) { _server_address = server_address; _connect_to_ps_server(); } - private void _connect_to_ps_server() - { + private void _connect_to_ps_server() { bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server: {_server_address}"); ClientToServerManagedTCPConnector.connect_to(_server_address, this, TimeSpan.FromSeconds(1)); } #region IPubSubServerConnection Members - public void sendWithConfirmation(byte[] data, ref string conf_id) - { + public void sendWithConfirmation(byte[] data, ref string conf_id) { bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} myConnection : {_connection != null}"); - if (_connection != null) - { - _connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) => - { + if (_connection != null) { + _connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) => { if (_on_send_onfirm_callback != null) _on_send_onfirm_callback(msg_id, confirmed); }); - } - else - { + } else { bts.Log.Gen.warn($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} no connection, buffering send"); conf_id = Guid.NewGuid().ToString(); _negative_unsent_confs.enqueue(conf_id); } } - public void send(byte[] data) - { + public void send(byte[] data) { bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_address} data: {data.Length} myConnection : {_connection != null}"); - if (_connection != null) - { + if (_connection != null) { bts.Log.Gen.info("PubSubServerManagedTCPConnection.send server: " + _server_address + "data: " + data.Length); _connection.send(data); } } - public void send(Stream str) - { + public void send(Stream str) { BinaryReader br = new BinaryReader(str); send(br.ReadBytes(Convert.ToInt32(str.Length - str.Position))); } - public void send(PubSubMessage msg) - { + public void send(PubSubMessage msg) { send(msg.to_stream()); } - public void send_with_confirmation(PubSubMessage msg, ref string conf_id) - { + public void send_with_confirmation(PubSubMessage msg, ref string conf_id) { var bytes = msg.to_bytes(); sendWithConfirmation(bytes, ref conf_id); } - public IList getUpdates(TimeSpan timeout, int maxElements) - { + public IList getUpdates(TimeSpan timeout, int maxElements) { return _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements); } - public string serverURL - { + public string serverURL { get { return _server_address.ToString(); } } - public string clientURL - { - get - { + public string clientURL { + get { string rv = ""; - lock (this) - { - if (_connection == null) - { + lock (this) { + if (_connection == null) { rv = ""; - } - else - { - if (PubSub3Config.instance.PS3_IPTranslation == null || PubSub3Config.instance.PS3_IPTranslation == String.Empty) - { + } else { + if (PubSub3Config.instance.PS3_IPTranslation == null || PubSub3Config.instance.PS3_IPTranslation == String.Empty) { string ip; int port; _connection.getSelfAddress(out ip, out port); rv = new BlutoSubAddress(ip, port).ToString(); - } - else - { + } else { rv = ""; } } @@ -143,18 +118,15 @@ namespace BCG } } - public void registerCallback(ConnectionStatusChange del) - { + public void registerCallback(ConnectionStatusChange del) { _conn_callback = del; } - public void registerDataCallback(DataDelegate del) - { + public void registerDataCallback(DataDelegate del) { _data_callback = del; } - public void registerSendConfirmationCallback(SendConfirmationDelegate del) - { + public void registerSendConfirmationCallback(SendConfirmationDelegate del) { _on_send_onfirm_callback = del; } @@ -163,21 +135,17 @@ namespace BCG #region IDisposable Members private bool _is_disposed = false; - public void Dispose() - { + public void Dispose() { bts.Log.Gen.info("PubSubServerManagedTCPConnection.Dispose"); //throw new Exception("The method or operation is not implemented."); - if (_connection != null) - { + if (_connection != null) { _connection.Dispose(); } _is_disposed = true; } - public bool isDisposed - { - get - { + public bool isDisposed { + get { return _is_disposed; } @@ -193,12 +161,10 @@ namespace BCG /// (long) is safe for multi threaded operations /// /// - public void onData(RemoteMessage message) - { + public void onData(RemoteMessage message) { bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData message: {message}"); // TODO check for disconenction - switch (message.messageType) - { + switch (message.messageType) { case RemoteMessageType.Data: if (_data_callback != null) _data_callback(message.remoteConnectionInfo.ToString(), message.message); @@ -219,17 +185,18 @@ namespace BCG break; } - if (message.messageType == RemoteMessageType.Data) - { - if (_data_callback != null) - _data_callback(message.remoteConnectionInfo.ToString(), message.message); - else - _in_data_queue.enqueue(message.message); - } - else - { - _last_hb = DateTime.Now; - } + // this stupid shit, caused duplicating og messages and 2 days of wild goose chase + // if (message.messageType == RemoteMessageType.Data) + // { + // if (_data_callback != null) + // _data_callback(message.remoteConnectionInfo.ToString(), message.message); + // else + // _in_data_queue.enqueue(message.message); + // } + // else + // { + // _last_hb = DateTime.Now; + // } } #endregion @@ -241,15 +208,13 @@ namespace BCG /// except onData) /// /// - public void onConnection(ITCPConnection connection) - { + public void onConnection(ITCPConnection connection) { string ip; int port; connection.getPeerAddress(out ip, out port); BlutoSubAddress add = new BlutoSubAddress(ip, port); bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onConnection add: {add.ToString()} connection type: {connection.GetType()}"); - if (add.ToString() == _server_address.ToString()) - { + if (add.ToString() == _server_address.ToString()) { _last_hb = DateTime.Now; _new_connection = connection; } @@ -257,39 +222,31 @@ namespace BCG #endregion - void _dispose_and_reconnect_if_missing_hb() - { - if (!_connection.isDisposed) - { + void _dispose_and_reconnect_if_missing_hb() { + if (!_connection.isDisposed) { TimeSpan hb_interval = DateTime.Now - _last_hb; bool hb_missed = hb_interval > _connection_drop_threshold; - if (hb_missed) - { + if (hb_missed) { if (_conn_callback != null) _conn_callback(_server_address.ToString(), false); bts.Log.Gen.info("PubSubServerManagedTCPConnection._dispose_and_reconnect_if_missing_hb disposing connecion: {myServerAddress}"); _connection.Dispose(); _connect_to_ps_server(); } - } - else - { + } else { // already disposed, nothing to do } } - void _check_for_new_connection() - { - if (_new_connection != null) - { + void _check_for_new_connection() { + if (_new_connection != null) { _connection = _new_connection; _new_connection = null; // since we are using its presence as a marker of nw connections bts.Log.Gen.info("PubSubServerManagedTCPConnection._check_for_new_connection assigning new connection"); _connection.register(this); if (_conn_callback != null) _conn_callback(_server_address.ToString(), true); - if (_on_send_onfirm_callback != null) - { + if (_on_send_onfirm_callback != null) { var msg_ids = _negative_unsent_confs.dequeueAll(); foreach (var id in msg_ids) _on_send_onfirm_callback(id, false); @@ -304,8 +261,7 @@ namespace BCG /// the design only allows for onConnection to be on a different /// thread. /// - public void doWork() - { + public void doWork() { //_dispose_and_reconnect_if_missing_hb(); _check_for_new_connection(); ///check for heartbeats and let clients know in diff --git a/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs b/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs index f77daf791..a8acd1250 100644 --- a/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs +++ b/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs @@ -304,6 +304,7 @@ namespace BCG { } private void myCollectMissingHeartbeatConnections() { + return; // disabling HBs for now to simplify things for the demo if (DateTime.Now - myLastHeartbeatDeadCollectionTime > mySubscriberDropThreshold) { myLastHeartbeatDeadCollectionTime = DateTime.Now; ///these are all the hbs received bw the last @@ -321,10 +322,9 @@ namespace BCG { if (!indexedHbs.ContainsKey(clientURL)) deadClients.Add(clientURL); foreach (string deadClient in deadClients) { - string disposeMsg = String.Format("Collecting missing HB connections : {0} is dead due to missing hbs, Disposing it now, " + + string dispose_msg = String.Format("Collecting missing HB connections : {0} is dead due to missing hbs, Disposing it now, " + "connection count before deleting this = {1}", deadClient, myConnections.Count); - //BlueEventLog.logEvent(disposeMsg, PS3LogLevels.disconnection); - Console.WriteLine(disposeMsg); + bts.Log.Gen.error(dispose_msg); myConnections[deadClient].Dispose(); myConnections.Remove(deadClient); myDeadConnectionURLs.enqueue(deadClient); diff --git a/src/DataStreamSuite/PubSub/StaticPublisherCore.cs b/src/DataStreamSuite/PubSub/StaticPublisherCore.cs index 06f8e6d83..41352f3af 100644 --- a/src/DataStreamSuite/PubSub/StaticPublisherCore.cs +++ b/src/DataStreamSuite/PubSub/StaticPublisherCore.cs @@ -181,7 +181,7 @@ namespace BCG { public void publish(string tagType, string tag, Stream content) { PublisherCoreMessage msg = new PublisherCoreMessage(tagType, tag, content); - bts.Log.Gen.info($"StaticPublisherCore.publish msg: {msg}"); + bts.Log.Gen.info($"StaticPublisherCore.publish tag: {tag} msg: {msg} len: {content.Length - content.Position}"); myMainQueue.enqueue(msg); } @@ -357,7 +357,7 @@ namespace BCG { myState.publish(tag, tagType, content); if (!myCrushableTagsWaitingOnAck.ContainsKey(tag)) { bts.Log.Gen.info("Sending crushable tag: {tag}"); - mySendData(tag, tagType, content); + mySendData(tag, tagType, content, "myPublish"); } else { bts.Log.Gen.info("Could not send tag: {tag} since its waiting on an ack"); myCrushableTagsWaitingOnAck[tag].second++; @@ -374,8 +374,8 @@ namespace BCG { } // add header, possibly compress the data, and sendto appropriate pubsub - private void mySendData(string tag, string tagType, Stream content) { - bts.Log.Gen.info($"StaticPublisherCore.mySendData tag: {tag} tagType: {tagType}"); + private void mySendData(string tag, string tagType, Stream content, string reason) { + bts.Log.Gen.info($"StaticPublisherCore.mySendData tag: {tag} tagType: {tagType} reason: {reason} len: {content.Length - content.Position}"); long initPos = content.Position; // locate right pubsub for this tag IPubSubServerConnection serverConn = myGetOrBuildSender(tag); @@ -407,7 +407,6 @@ namespace BCG { serverConn.send_with_confirmation(ps_msg, ref refId); myCrushableTagsWaitingOnAck.Add(tag, new Pair(refId, 0)); } else { - //Console.WriteLine("StaticPublisherCore.mySend serverConn.send tag: " + tag + " server: " + serverConn.serverURL); // this is where the data is actually sent out var ps_msg = new PubSubMessage(pub_md, contentToPublish); bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}"); @@ -440,6 +439,7 @@ namespace BCG { ///in case of the server coming up, it publishes all the stored state ///for that server, back to it protected void myHandleServerReboot(string serverURL, bool connection) { + bts.Log.Gen.info($"StaticPublisherCore.myHandleServerReboot server_url: {serverURL}"); if (connection) { string thisPublisherInfo = ClientInfo.thisClient.processString; foreach (KeyValuePair p in myTagToSender) { @@ -448,7 +448,7 @@ namespace BCG { string tag = p.Key; myCrushableTagsWaitingOnAck.Clear(); byte[] data = myState.getContent(tag, out tagType); - mySendData(tag, tagType, new MemoryStream(data)); + mySendData(tag, tagType, new MemoryStream(data), "myHandleServerReboot"); //Stream content = new MemoryStream(data); //Stream contentToPublish = content; diff --git a/src/DataStreamSuite/PubSub/StaticSubscriberCore.cs b/src/DataStreamSuite/PubSub/StaticSubscriberCore.cs index 8d3c7c814..1911ae8d3 100644 --- a/src/DataStreamSuite/PubSub/StaticSubscriberCore.cs +++ b/src/DataStreamSuite/PubSub/StaticSubscriberCore.cs @@ -13,8 +13,7 @@ using mbts; using mbts.spark; //using BCG.Utilities.BlueConfiguration; -namespace BCG -{ +namespace BCG { /// /// This class serves as the subscrition interface to the pubsub. The only methods @@ -25,70 +24,56 @@ namespace BCG /// to the servers even though there might be so created by other instances of the SubscriberCore /// /// - public class StaticSubscriberCore : ISubscriberCore - { + public class StaticSubscriberCore : ISubscriberCore { private Dictionary myServerURLToSubCore = new Dictionary(); private IPubSubRouter myRouter; private HighPerformanceTimer myTimer = new HighPerformanceTimer(); private PubSubServerConnectionFactory myConnectionFactory = new PubSubServerConnectionFactory(true); public event PubSubConnectionChange onPubSubConnectionChange; - - + + #region constructors - public StaticSubscriberCore() - { + public StaticSubscriberCore() { //this defaults to a basic pubsubrouter if the config key is not filled out myInit(myGetRouter()); } - public StaticSubscriberCore(string siteName) - { + public StaticSubscriberCore(string siteName) { myInit(new PubSubRouter(PubSub3Config.instance.getRouting(siteName))); } - private IPubSubRouter myGetRouter() - {return new PubSubRouter(PubSub3Config.instance.getRouting());} + private IPubSubRouter myGetRouter() { return new PubSubRouter(PubSub3Config.instance.getRouting()); } - public StaticSubscriberCore(PubSubRoute[] routes) - { + public StaticSubscriberCore(PubSubRoute[] routes) { myInit(new PubSubRouter(routes)); } - public StaticSubscriberCore(IPubSubRouter router) - { + public StaticSubscriberCore(IPubSubRouter router) { myInit(router); } #endregion - private void myInit(IPubSubRouter router) - { + private void myInit(IPubSubRouter router) { myRouter = router; } - public ArrayList getUpdates() - { + public ArrayList getUpdates() { return getUpdates(10); } - public ArrayList getUpdates(TimeSpan timeout) - { + public ArrayList getUpdates(TimeSpan timeout) { return getUpdates(Convert.ToInt32(timeout.TotalMilliseconds)); } - public ArrayList getUpdates(int timeout) - { + public ArrayList getUpdates(int timeout) { //BCG.BlueTrace.WriteTraceMethodEnterW("StaticSubscriberCore.getUpdates"); - lock (myServerURLToSubCore) - { + lock (myServerURLToSubCore) { - if (myServerURLToSubCore.Count == 0) - { + if (myServerURLToSubCore.Count == 0) { return new ArrayList(); - } - else - { + } else { ///gets a time share for every subscriber and ///calls their updates myTimer.start(); @@ -96,6 +81,7 @@ namespace BCG List retVal = new List(); foreach (ISingleServerSubscriberCore subCore in myServerURLToSubCore.Values) retVal.AddRange(subCore.getUpdates(timeShare)); + _log_n_updates(retVal); myTimer.stop(); //if (myTimer.lastIntervalTime > 10) // BlueEventLog.logEvent("get updates taking " + myTimer.lastIntervalTime, BlueEventLevel.Error); @@ -104,30 +90,36 @@ namespace BCG return new ArrayList(retVal); } } - + } - private Dictionary myBlueCommTags = new Dictionary (); - - bool myIsBlueCommTag(string tag) - { - return false; + private IDictionary _n_update_map = new Dictionary(); + private void _log_n_updates(IEnumerable updates) { + foreach (PubSubUpdate u in updates) { + if (!_n_update_map.ContainsKey(u.tag)) { + _n_update_map.Add(u.tag, 0); + } + _n_update_map[u.tag] = _n_update_map[u.tag] + 1; + } + // log it now + foreach (var kv in _n_update_map) { + bts.Log.Gen.info($"StaticSubscrberCore tag: {kv.Key} n_updates: {kv.Value}"); + } } - public void subscribe(params string[] tags) - { - lock (myServerURLToSubCore) - { - foreach (string tag in tags) - { - if (myIsBlueCommTag(tag)) - { - } - else - { + private Dictionary myBlueCommTags = new Dictionary(); + + bool myIsBlueCommTag(string tag) { + return false; + } + + public void subscribe(params string[] tags) { + lock (myServerURLToSubCore) { + foreach (string tag in tags) { + if (myIsBlueCommTag(tag)) { + } else { string serverURL = myRouter.getSubURLForTag(tag); - if (!myServerURLToSubCore.ContainsKey(serverURL)) - { + if (!myServerURLToSubCore.ContainsKey(serverURL)) { // BlueEventLog.logEvent("Creating a SingleServerSubscriberCore for " + serverURL, BlueEventLevel.Information); myServerURLToSubCore.Add(serverURL, new SingleServerSubscriberCore(serverURL, myConnectionFactory)); myServerURLToSubCore[serverURL].onPubSubConnectionChange += myHandleConnectionChange; @@ -138,28 +130,22 @@ namespace BCG } } } - public void myHandleConnectionChange (bool isConnected, string connection) - { + public void myHandleConnectionChange(bool isConnected, string connection) { if (onPubSubConnectionChange != null) onPubSubConnectionChange(isConnected, connection); } - public void unsubscribe(params string[] tags) - { - lock (myServerURLToSubCore) - { + public void unsubscribe(params string[] tags) { + lock (myServerURLToSubCore) { - foreach (string tag in tags) - { + foreach (string tag in tags) { if (myIsBlueCommTag(tag)) myServerURLToSubCore["bluecomm"].unsubscribe(tag); - else - { + else { string serverURL = myRouter.getSubURLForTag(tag); - if (!myServerURLToSubCore.ContainsKey(serverURL)) - { + if (!myServerURLToSubCore.ContainsKey(serverURL)) { // BlueEventLog.logEvent("Creating a SingleServerSubscriberCore for " + serverURL, BlueEventLevel.Information); myServerURLToSubCore.Add(serverURL, new SingleServerSubscriberCore(serverURL, myConnectionFactory)); } @@ -168,15 +154,13 @@ namespace BCG } } } - public IPubSubRouter getRouter() - { + public IPubSubRouter getRouter() { return myRouter; } #region IDisposable Members - public void Dispose() - { + public void Dispose() { foreach (ISingleServerSubscriberCore sssc in myServerURLToSubCore.Values) sssc.Dispose(); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs index 230398791..02824ad5c 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs @@ -40,7 +40,7 @@ namespace BCG { IReadOnlyDictionary _cached_state = null; MarketDataClient _md_client; HighPerformanceTimer _timer = new HighPerformanceTimer(); - private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("bbo", + private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("books", Tuple.Create("id", TypeCode.Int32), Tuple.Create("sec_key", TypeCode.UInt32), Tuple.Create("level", TypeCode.UInt32), diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs index 823c4ca3e..82586a5dd 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs @@ -1,20 +1,26 @@ -namespace BCG { +namespace BCG +{ using bts.api.orderman; - enum TSGroupByOperation { + enum TSGroupByOperation + { OHLC = 0, Statistics } [DataStreamUserRespresentation("timeseriesanalysis")] - public class TimeSeriesGroupByStream : ViewStream { - public TimeSeriesGroupByStream(ViewStream md_str, string timestamp_col, string output_col, string operation) { - ViewStream rs = new NonIncGroupByStream(md_str, new OHLCNonIncGroupByEvaluator(timestamp_col, output_col)); + public class TimeSeriesGroupByStream : ViewStream + { + public TimeSeriesGroupByStream(ViewStream md_str, string timebucket_col, string output_col, string operation) + { + ViewStream rs = new NonIncGroupByStream(md_str, new OHLCNonIncGroupByEvaluator(timebucket_col, output_col)); mySetRealSource(rs); } } - public class OHLC { - public OHLC(double open_args, double high_args, double low_args, double close_args) { + public class OHLC + { + public OHLC(double open_args, double high_args, double low_args, double close_args) + { open = open_args; high = high_args; low = low_args; @@ -22,36 +28,47 @@ namespace BCG { } public readonly double open, high, low, close; } - public class OHLCNonIncGroupByEvaluator : INonIncGroupByEvaluator { + public class OHLCNonIncGroupByEvaluator : INonIncGroupByEvaluator + { private string _timestamp_col; private string _output_col; private IBlueTableSchema _out_schema; - public OHLCNonIncGroupByEvaluator(string timestamp_col, string output_col) { + public OHLCNonIncGroupByEvaluator(string timestamp_col, string output_col) + { _timestamp_col = timestamp_col; _output_col = output_col; } - string open_col { - get { + string open_col + { + get + { return $"{_output_col}_open"; ; } } - string high_col { - get { + string high_col + { + get + { return $"{_output_col}_high"; ; } } - string low_col { - get { + string low_col + { + get + { return $"{_output_col}_low"; ; } } - string close_col { - get { + string close_col + { + get + { return $"{_output_col}_close"; ; } } - private void _build_schema(IBlueTableSchema input_schema) { + private void _build_schema(IBlueTableSchema input_schema) + { List> new_cols = new List>(); ColumnInfo ci = input_schema[_output_col]; new_cols.Add(Tuple.Create(ci.colType, open_col)); @@ -62,23 +79,30 @@ namespace BCG { _out_schema = BlueDataUtility.add_columns(input_schema, new_cols.ToArray()); } - public IBlueRowCollection evaluate(IDictionary> grouped_rows) { - if (_out_schema == null) { - if (grouped_rows.Count > 0) { - foreach (var kv in grouped_rows) { + public IBlueRowCollection evaluate(IDictionary> grouped_rows) + { + if (_out_schema == null) + { + if (grouped_rows.Count > 0) + { + foreach (var kv in grouped_rows) + { var rows = kv.Value; - if (rows != null && rows.Count > 0) { + if (rows != null && rows.Count > 0) + { _build_schema(rows.ElementAt(0).schema); } } } } BlueTable rv = new BlueTable(_out_schema); - foreach (var kv in grouped_rows) { + foreach (var kv in grouped_rows) + { int id = kv.Key; IEnumerable rows = kv.Value; // sort rows on time to compute o,c correctly?? - if (rows != null && rows.Count() > 0) { + if (rows != null && rows.Count() > 0) + { IBlueRow reprow = rows.ElementAt(0); IEnumerable vals = BlueDataUtility.get_covalues(rows, _output_col); OHLC ohlc = compute_ohlc(vals); @@ -94,11 +118,13 @@ namespace BCG { return rv; } - public string[] get_group_by_cols() { + public string[] get_group_by_cols() + { return new string[] { _timestamp_col }; } - static OHLC compute_ohlc(IEnumerable prices) { + static OHLC compute_ohlc(IEnumerable prices) + { double o = prices.ElementAt(0); double c = prices.ElementAt(prices.Count() - 1); double l = prices.Min(); @@ -106,11 +132,30 @@ namespace BCG { return new(o, h, l, c); } } - [DataStreamUserRespresentation("ohlc")] - public class OHLCStream : ViewStream { + [DataStreamUserRespresentation("RSI")] + public class RSIStream : ViewStream + { // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for - public OHLCStream(ViewStream md_str, string timestamp_col, string output_col) { - ViewStream rs = new TimeSeriesGroupByStream(md_str, timestamp_col, output_col, TSGroupByOperation.OHLC.ToString()); + public RSIStream(ViewStream md_str, string timebucket_column, string output_col) + { + + } + } + [DataStreamUserRespresentation("Bollinger")] + public class BollingerBandStream : ViewStream + { + // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for + public BollingerBandStream(ViewStream md_str, string timebucket_column, string output_col) + { + } + } + [DataStreamUserRespresentation("OHLC")] + public class OHLCStream : ViewStream + { + // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for + public OHLCStream(ViewStream md_str, string timebucket_column, string output_col) + { + ViewStream rs = new TimeSeriesGroupByStream(md_str, timebucket_column, output_col, TSGroupByOperation.OHLC.ToString()); mySetRealSource(rs); } } diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimeSalesStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimeSalesStream.cs index adcd48b33..acad6ab0d 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimeSalesStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimeSalesStream.cs @@ -1,4 +1,5 @@ -namespace BCG { +namespace BCG +{ using BCG.Utilities.General; using bts.api; @@ -19,7 +20,8 @@ namespace BCG { using SecurityKey = System.UInt32; [DataStreamUserRespresentation("timesales")] - public class TimeSalesStream : ViewStream, IWorker { + public class TimeSalesStream : ViewStream, IWorker + { private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500); private DateTime _last_poll_time = DateTime.MinValue; private ViewStreamDatabase _vsd; @@ -28,37 +30,43 @@ namespace BCG { MarketDataClient _md_client; private bool _dirty = false; private ConcurrentQueue _cached_trades = new ConcurrentQueue(); - private BlueTable _table = new BlueTable(BlueDataUtility.form_schema("instruments", + private readonly int _buffer_size; + private IBlueTableSchema _schema = BlueDataUtility.form_schema("TimeSales", Tuple.Create("id", TypeCode.Int32), Tuple.Create("sec_key", TypeCode.UInt32), Tuple.Create("sec", TypeCode.String), - Tuple.Create("time", TypeCode.Int64), + Tuple.Create("time", TypeCode.DateTime), Tuple.Create("price", TypeCode.Double), Tuple.Create("qty", TypeCode.UInt32), - Tuple.Create("aggressor", TypeCode.String) - )); - public TimeSalesStream() - : this(ViewStreamDatabase.instance) { + Tuple.Create("aggressor", TypeCode.String)); + public TimeSalesStream(int buffer_size) + : this(buffer_size, ViewStreamDatabase.instance) + { } - public TimeSalesStream(ViewStreamDatabase vsd) { + public TimeSalesStream(int buffer_size, ViewStreamDatabase vsd) + { + _buffer_size = buffer_size; bts.Log.Gen.info("TimeSalesStream.cons"); _vsd = vsd; _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5))); } - protected override void myOnFirstSubscriber() { + protected override void myOnFirstSubscriber() + { _vsd.addWorker(this); _md_client.trades(new TradesSubscriptionFilter()).Subscribe(_on_trades); //_md_client.trades(null).Subscribe(_on_trades); } - private void _log_updates(IList trades) { + private void _log_updates(IList trades) + { bts.Log.Gen.info($"TimeSalesStream update sz: {trades.Count}"); } - private void _on_trades(IList trades) { + private void _on_trades(IList trades) + { _dirty = true; _log_updates(trades); // just cacching in here, will populate the rows in _poll_scan and fire the datapacket @@ -67,23 +75,35 @@ namespace BCG { _cached_trades.Enqueue(t); } - protected override void myOnLastSubscriber() { + protected override void myOnLastSubscriber() + { _vsd.removeWorker(this); } - private void _scan_trades() { - BlueTable added_rows = new BlueTable(_table.schema); - foreach (TradeInfo tinf in _cached_trades) { - IBlueRow r = new BlueRow(_table.schema, _id++, tinf.security.Id, tinf.security.ToString(), tinf.trade.Time, tinf.trade.Mantissa, tinf.trade.Size, tinf.trade.AggressorSide.ToString()); - added_rows.addRow(r); + private void _scan_trades() + { + Func to_dt = (tinf) => + { + return new mbts.utils.LLDateTime((ulong)tinf.trade.Time).to_datetime().ToLocalTime(); + }; + + List trades = new List(_cached_trades); + var latest_n_trades = trades.OrderByDescending(t => (ulong)t.trade.Time).Take(_buffer_size).ToList(); + BlueTable new_state = new BlueTable(_schema); + foreach (TradeInfo tinf in Enumerable.Reverse(latest_n_trades)) + { + // TODO, time + IBlueRow r = new BlueRow(_schema, _id++, tinf.security.Id, tinf.security.ToString(), to_dt(tinf), tinf.trade.Mantissa * Math.Pow(10, tinf.trade.Exponent), tinf.trade.Size, tinf.trade.AggressorSide.ToString()); + new_state.addRow(r); } - _table.addRows(added_rows); - if (added_rows.Count > 0) - myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema))); + DataPacket dp = BlueDataUtility.build_dp_inc(new_state, myLastValue as DataPacket); + myFireEvent(dp); } - public void doWork() { - if (_dirty && DateTime.Now - _last_poll_time > _poll_period) { + public void doWork() + { + if (_dirty && DateTime.Now - _last_poll_time > _poll_period) + { _last_poll_time = DateTime.Now; _dirty = false; _scan_trades(); diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubServerTagStatsStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubServerTagStatsStream.cs index 2c5dbcb31..3d982bc44 100644 --- a/src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubServerTagStatsStream.cs +++ b/src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubServerTagStatsStream.cs @@ -45,7 +45,7 @@ namespace BCG #region constructors to be used by the static factory method - private PubSubServerTagStatsStream(string serverURL) + public PubSubServerTagStatsStream(string serverURL) : this(ViewStreamDatabase.instance, serverURL) { }