diff --git a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs index c92fcf72b..6f6ca4b26 100644 --- a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs @@ -24,7 +24,6 @@ namespace bts { var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket); myFireEvent(dp_out); - } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs index fea7e8570..d54f012da 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs @@ -5,18 +5,21 @@ using System.Linq; using bts; -namespace BCG { +namespace BCG +{ // This class is responsible for mantaining the state fo a table. It gets the // bytes from the pubsubserver via the handle_updates call (called by VSD), deserializes // updates the state of the table, and fires the TableUpdateDelegate. - public class TableAccumulator2 : ITableAccumulator { + public class TableAccumulator2 : ITableAccumulator + { private readonly string _tag; private readonly ISubscriberCore _subscriber_core; IStreamMsgDeserilizer _deser = new StreamIODeserializationController(); private BlueTable _table; private TableUpdateDelegate _on_update_delegate; - public TableAccumulator2(string tag, ISubscriberCore sub_core) { + public TableAccumulator2(string tag, ISubscriberCore sub_core) + { _tag = tag; _subscriber_core = sub_core; } @@ -25,10 +28,13 @@ namespace BCG { //public bool is_active() { return _on_update_delegate != null; } - public event TableUpdateDelegate on_update { - add { + public event TableUpdateDelegate on_update + { + add + { Log.Gen.info("TableAccumulator2.on_update.add {_tag} subscribed"); - if (_on_update_delegate == null) { + if (_on_update_delegate == null) + { //ie first subscription Log.Gen.info("TableAccumulator2.on_update {_tag} subscribed for the first time"); _subscriber_core.subscribe(_tag); @@ -38,14 +44,17 @@ namespace BCG { if (_table != null) value(_table, _table, new BlueTable(_table.schema)); } - remove { + remove + { Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed"); - if (_on_update_delegate != null) { + if (_on_update_delegate != null) + { _on_update_delegate = Delegate.Remove(_on_update_delegate, value) as TableUpdateDelegate; int n_subs = _on_update_delegate == null ? 0 : _on_update_delegate.GetInvocationList().Length; Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed n_subs: {n_subs}"); - if (_on_update_delegate == null) { + if (_on_update_delegate == null) + { _subscriber_core.unsubscribe(_tag); // TODO //_table = null; @@ -55,16 +64,19 @@ namespace BCG { } } private int _update_no = 0; - public void handle_updates(IEnumerable msgs) { + public void handle_updates(IEnumerable msgs) + { if (msgs == null || msgs.Count() == 0) return; Log.Gen.info($"TableAccumulator2.handle_updates _update_no: {_update_no++} msgs: {msgs.Count()} msg_0: {msgs.ElementAt(0).Length}"); IBlueRowCollection table_0 = _table == null ? null : new BlueTable(_table); - foreach (var msg in msgs) { + foreach (var msg in msgs) + { bts.StreamIOMessage str_msg = _deser.deserialize(msg); var state_txt = _table == null ? "NULL" : BlueDataUtility.id_string(_table); Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} sz: {msg.Length} str_msg: {str_msg} _state: {state_txt}"); - switch (str_msg.op) { + switch (str_msg.op) + { case bts.StreamIOMessage.StreamOp.CREATE: _table = new BlueTable(str_msg.create_message.rows); break; @@ -77,11 +89,13 @@ namespace BCG { // this could be made faster if we just do this on the CREATE message, and pass-through // what we get on the MULTIPLE_REMOVE_ADD IBlueRowCollection d_rows, a_rows; - BlueDataUtility.compute_differences_id_based(_table, table_0, out d_rows, out a_rows); + //BlueDataUtility.compute_differences_id_based(_table, table_0, out d_rows, out a_rows); + BlueDataUtility.compute_differences_fast(_table, table_0, out d_rows, out a_rows); _on_update_delegate(_table, a_rows, d_rows); } - public void update(Stream[] streams) { + public void update(Stream[] streams) + { } #endregion diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json index 1a8449f08..8c5196f98 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json +++ b/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json @@ -1,6 +1,6 @@ { "user": "bts", - "sparkd": "http://localhost:8000/config", + "sparkd": "http://192.168.1.80:8000/config", "pubsub": { "milliseconds_between_sub_info_updates": 1000, "milliseconds_between_time_slice_updates": 5000, @@ -18,14 +18,14 @@ "publisher_routes": [ { "name": "default", - "server_address": "bluto://localhost:12345", + "server_address": "bts-cloud://127.0.0.1:12345", "tag": ".*" } ], "subscriber_routes": [ { "name": "default", - "server_address": "bluto://localhost:12345", + "server_address": "bts-cloud://127.0.0.1:12345", "tag": ".*" } ], @@ -106,4 +106,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Streams/TablesMetaDataStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/TableMetaDataStream.cs similarity index 56% rename from src/DataStreamSuite/SharedStreams/System/PS3/Streams/TablesMetaDataStream.cs rename to src/DataStreamSuite/SharedStreams/System/PS3/TableMetaDataStream.cs index 3ca199965..49ab64a91 100644 --- a/src/DataStreamSuite/SharedStreams/System/PS3/Streams/TablesMetaDataStream.cs +++ b/src/DataStreamSuite/SharedStreams/System/PS3/TableMetaDataStream.cs @@ -1,23 +1,27 @@ - - -namespace bts { +namespace bts +{ using BCG; using BCG.Utilities.General; - public class TableMetaDataStream : ViewStream, IWorker { - private class TableState { + public class TableMetaDataStream : ViewStream, IWorker + { + private class TableState + { public ViewStream table_stream; public IBlueTableSchema schema = null; public bool processed = false; - public TableState(ViewStream table_stream_) { + public TableState(ViewStream table_stream_) + { table_stream = table_stream_; table_stream.onChange += _handle_table_update; } - private void _handle_table_update(object dp_obj) { + private void _handle_table_update(object dp_obj) + { DataPacket dp = dp_obj as DataPacket; schema = dp.schema; } - public void unsub() { + public void unsub() + { table_stream.onChange -= _handle_table_update; } } @@ -34,62 +38,74 @@ namespace bts { private BlueTable _state; public TableMetaDataStream() : this(ViewStreamDatabase.instance) { } - public TableMetaDataStream(ViewStreamDatabase vsd) { + public TableMetaDataStream(ViewStreamDatabase vsd) + { _vsd = vsd; _base_stream = new PubSubServerStateStream(_vsd); _state = new BlueTable(_schema); } - protected override void myOnFirstSubscriber() { + protected override void myOnFirstSubscriber() + { _base_stream.onChange += _handle_base_stream; _vsd.addWorker(this); } - protected override void myOnLastSubscriber() { + protected override void myOnLastSubscriber() + { _base_stream.onChange -= _handle_base_stream; _vsd.removeWorker(this); } - public void doWork() { + public void doWork() + { bool changed = false; - foreach (TableState ts in _table_map.Values) { - if (!ts.processed) { - if (ts.schema != null) { - foreach (ColumnInfo ci in ts.schema.columnInfo) { - int id = _state.Count; - var table_name = ts.schema.tableName; - var col_idx = ci.colNo; - var col_name = ci.colName; - TypeCode type_code = BlueDataUtility.get_typecode(ci.colType); - BlueRow nr = new BlueRow(_schema, id, table_name, col_idx, Convert.ToInt32(type_code), - col_name); - changed = true; - _state.addRow(nr); - } + foreach (TableState ts in _table_map.Values) + { + if (!ts.processed && ts.schema != null) + { + foreach (ColumnInfo ci in ts.schema.columnInfo) + { + int id = _state.Count; + var table_name = ts.schema.tableName; + var col_idx = ci.colNo; + var col_name = ci.colName; + TypeCode type_code = BlueDataUtility.get_typecode(ci.colType); + BlueRow nr = new BlueRow(_schema, id, table_name, col_idx, Convert.ToInt32(type_code), + col_name); + changed = true; + _state.addRow(nr); } + ts.processed = true; ts.unsub(); } } - if (changed) { + if (changed) + { var dp_out = BlueDataUtility.build_dp_inc(_state, myLastValue as DataPacket); myFireEvent(dp_out); } } - private void _handle_base_stream(object dp_obj) { + private void _handle_base_stream(object dp_obj) + { DataPacket dp = dp_obj as DataPacket; - foreach (IBlueRow r in dp.newRows) { + foreach (IBlueRow r in dp.newRows) + { string tag = r.get("tag"); - if (tag.StartsWith("DS_")) { + if (tag.StartsWith("DS_")) + { _handle_table(tag); } } } - private void _handle_table(string tag_name) { + private void _handle_table(string tag_name) + { Log.assert(tag_name.StartsWith("DS_")); string table_name = tag_name.Remove(0, 3); TableState ts = null; - if (!_table_map.TryGetValue(table_name, out ts)) { + if (!_table_map.TryGetValue(table_name, out ts)) + { ts = new TableState(new TableStream(table_name, _vsd)); _table_map.Add(table_name, ts); }