diff --git a/src/DataStreamSuite/DataStream/DataStream.csproj b/src/DataStreamSuite/DataStream/DataStream.csproj index 960706a61..577b35b5f 100644 --- a/src/DataStreamSuite/DataStream/DataStream.csproj +++ b/src/DataStreamSuite/DataStream/DataStream.csproj @@ -15,6 +15,7 @@ + diff --git a/src/DataStreamSuite/DataStream/Primitives/KeyValue/KeyValueToRelationalStream.cs b/src/DataStreamSuite/DataStream/Primitives/KeyValue/KeyValueToRelationalStream.cs new file mode 100644 index 000000000..2a2fed4a4 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/KeyValue/KeyValueToRelationalStream.cs @@ -0,0 +1,31 @@ +namespace BCG { + using System.Collections.Generic; + + public class KeyValueToRelationalStream : ViewStream { + private readonly ViewStream _kv_str; + private readonly string _doc_id_col, _key_col, _val_col; + public KeyValueToRelationalStream(ViewStream kv_str, string doc_id_col, string key_col, string val_col) { + _kv_str = kv_str; + _doc_id_col = doc_id_col; + _key_col = key_col; + _val_col = val_col; + } + + protected override void myOnFirstSubscriber() { + _kv_str.onChange += _on_str_update; + } + + protected override void myOnLastSubscriber() { + _kv_str.onChange -= _on_str_update; + } + + private void _on_str_update(object dp_obj) { + DataPacket dp = dp_obj as DataPacket; + var rows = dp.newRows; + var rv = BlueDataUtility.keyvalue_to_relational(rows, _doc_id_col, _key_col, _val_col); + var dp_out = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket, true); + myFireEvent(dp_out); + } + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/WebScrapingStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/WebScrapingStream.cs new file mode 100644 index 000000000..9f70da850 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/WebScrapingStream.cs @@ -0,0 +1,121 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Security.Cryptography.X509Certificates; + +using BCG; +using BCG.Utilities.General; + +using HtmlAgilityPack; + +namespace BCG { + class Program { + static void Main() { + string url = "http://quotes.toscrape.com"; + List quotes = ScrapeQuotes(url); + + Console.WriteLine("Quotes from http://quotes.toscrape.com:"); + foreach (var quote in quotes) { + Console.WriteLine(quote); + } + } + + static List ScrapeQuotes(string url) { + List quotes = new List(); + + // Create a new HtmlWeb instance + HtmlWeb web = new HtmlWeb(); + + try { + // Load the HTML document from the specified URL + HtmlDocument document = web.Load(url); + + // Extract quotes from the HTML document + var quoteNodes = document.DocumentNode.SelectNodes("//span[@class='text']"); + if (quoteNodes != null) { + foreach (var quoteNode in quoteNodes) { + string quote = quoteNode.InnerText.Trim(); + quotes.Add(quote); + } + } + } catch (Exception ex) { + Console.WriteLine($"Error: {ex.Message}"); + } + + return quotes; + } + } + + public class WebScapperStream : ViewStream, IWorker { + private readonly string _url; + private readonly string _xpath; + private readonly ViewStreamDatabase _vsd; + private IDictionary _line_id_map = new Dictionary(); + private readonly IBlueTableSchema _schema; + + public WebScapperStream() + : + this("http://quotes.toscrape.com", "//span[@class='text']", ViewStreamDatabase.instance) { } + public WebScapperStream(string url, string xpath) + : this(url, xpath, ViewStreamDatabase.instance) { } + public WebScapperStream(string url, string xpath, ViewStreamDatabase vsd) { + bts.Log.Gen.info($"WebScapperStream cons url: {url} xpath: {xpath}"); + _schema = BlueDataUtility.form_schema($"{url}_{xpath}", + Tuple.Create("id", TypeCode.Int32), + Tuple.Create("url", TypeCode.UInt32), + Tuple.Create("line", TypeCode.String)); + _url = url; + _xpath = xpath; + _vsd = vsd; + } + + protected override void myOnFirstSubscriber() { + MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); + } + + protected override void myOnLastSubscriber() { + MasterWorker.getInstance(false).removeWorker(this); + } + + public void doWork() { + var lines = scrape_lines(_url, _xpath); + Func get_id = (line) => { + int rv = 0; + if (!_line_id_map.TryGetValue(line, out rv)) + _line_id_map.Add(line, _line_id_map.Count); + return rv; + }; + BlueTable nrows = new BlueTable(_schema); + foreach (var l in lines) { + bts.Log.Gen.info($"line: {l}"); + var id = get_id(l); + BlueRow nr = new BlueRow(_schema, get_id(l), _url, l); + nrows.addRow(nr); + } + var dp = BlueDataUtility.build_dp_inc(nrows, myLastValue as DataPacket, true); + myFireEvent(dp); + } + + static List scrape_lines(string url, string xpath) { + List quotes = new List(); + // Create a new HtmlWeb instance + HtmlWeb web = new HtmlWeb(); + try { + // Load the HTML document from the specified URL + HtmlDocument document = web.Load(url); + // Extract quotes from the HTML document + var quote_nodes = document.DocumentNode.SelectNodes(xpath); + if (quote_nodes != null) { + foreach (var qn in quote_nodes) { + string quote = qn.InnerText.Trim(); + quotes.Add(quote); + } + } + } catch (Exception ex) { + Console.WriteLine($"Error: {ex.Message}"); + } + + return quotes; + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs index bb2026f94..52015e568 100644 --- a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs @@ -9,42 +9,9 @@ using System.Runtime.CompilerServices; using mbts.spark; -class Program { - static void Main() { - // Set the connection string and database name - string connectionString = "mongodb://localhost:27017"; - string databaseName = "your_database_name"; - string collectionName = "your_collection_name"; - - // Create a MongoClient - MongoClient client = new MongoClient(connectionString); - - // Access the database - IMongoDatabase database = client.GetDatabase(databaseName); - - // Access the collection - IMongoCollection collection = database.GetCollection(collectionName); - - // Fetch all documents from the collection - var documents = collection.Find(Builders.Filter.Empty).ToCursor(); - - // Iterate through the documents and print values - foreach (var document in documents.ToEnumerable()) { - // Assuming your document structure has fields named "Field1", "Field2", etc. - Console.WriteLine("Field1: " + document["Field1"]); - Console.WriteLine("Field2: " + document["Field2"]); - // Add more fields as needed - - // Print a separator between documents - Console.WriteLine(new string('-', 20)); - } - } -} - - namespace BCG { - [DataStreamUserRespresentation("mongodb")] - public class MongoDBStream : ViewStream, IWorker { + [DataStreamUserRespresentation("mongodb_sql")] + public class MongoDBSQLStream : ViewStream, IWorker { private ViewStreamDatabase _vsd; private readonly string _connection_string, _db_name, _collection_name; private MongoClient _client; @@ -55,11 +22,11 @@ namespace BCG { private IDictionary _id_map = new Dictionary(); // mongodb://192.168.1.80:27017 - public MongoDBStream(string connection_string, string db_name, string collection_name) + public MongoDBSQLStream(string connection_string, string db_name, string collection_name) : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) { } - public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) { + public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) { bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " + $"db_name: {db_name}"); _connection_string = connection_string; @@ -139,4 +106,107 @@ namespace BCG { } } + [DataStreamUserRespresentation("mongodb")] + public class MongoDBStream : ViewStream, IWorker { + private ViewStreamDatabase _vsd; + private readonly string _connection_string, _db_name, _collection_name; + private MongoClient _client; + private BlueTable _table; + + // Access the database + private IMongoDatabase _database; + private IDictionary _id_map = new Dictionary(); + // mongodb://192.168.1.80:27017 + + public MongoDBStream(string connection_string, string db_name, string collection_name) + : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) { + + } + public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) { + bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " + + $"db_name: {db_name}"); + _connection_string = connection_string; + _db_name = db_name; + _collection_name = collection_name; + _vsd = vsd; + _client = new MongoClient(_connection_string); + _database = _client.GetDatabase(_db_name); + } + + protected override void myOnFirstSubscriber() { + MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); + } + + protected override void myOnLastSubscriber() { + MasterWorker.getInstance(false).removeWorker(this); + } + + public void doWork() { + _read_mogodb(); + } + + private void _read_mogodb() { + HighPerformanceTimer timer = new HighPerformanceTimer(); + timer.start(); + // Access the collection + IMongoCollection collection = _database.GetCollection(name: _collection_name); + + // Fetch all documents from the collection + var documents = collection.Find(Builders.Filter.Empty).ToCursor(); + _table = _table == null ? null : new BlueTable(_table.schema); + // Iterate through the documents and print values + foreach (var document in documents.ToEnumerable()) { + bts.Log.Gen.info($"document: {document} json: {document.ToJson()}"); + if (_table == null) { + _init_table(document); + } + _add_doc(document); + } + timer.stop(); + bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs"); + timer.start(); + var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true); + timer.stop(); + bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs"); + if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0) + return; + myFireEvent(dp); + } + + private void _init_table(BsonDocument doc_row) { + bts.Log.assert(_table == null); + var schema = BlueDataUtility.form_schema($"{_database}_{_table}", + Tuple.Create("id", TypeCode.Int32), + Tuple.Create("doc_id", TypeCode.String), + Tuple.Create("key", TypeCode.String), + Tuple.Create("value", TypeCode.String)); + _table = new BlueTable(schema); + } + + private void _add_doc(BsonDocument doc) { + var doc_id_bson = doc["_id"]; + var doc_id = doc_id_bson.ToString(); + Func get_id = (string doc_id, string key) => { + string hash_key = $"{doc_id}_{key}"; + int id = 0; + if (!_id_map.TryGetValue(hash_key, out id)) { + id = _id_map.Count; + _id_map.Add(hash_key, id); + } + return id; + }; + foreach (var n in doc.Names) { + string key = n; + var id = get_id(doc_id, key); + MutableBlueRow nr = new MutableBlueRow(_table.schema); + var val = doc[n].ToString(); + nr.setValue("id", id); + nr.setValue("doc_id", doc_id); + nr.setValue("key", key); + nr.setValue("value", val); + _table.addRow(nr); + } + } + } + } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs index 10f5b3ba2..a29191e2a 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs @@ -13,6 +13,7 @@ using System.Linq.Expressions; //using mbts.spark; using mbts; using BCG.Extensions; +using System.Runtime.CompilerServices; namespace BCG { public static class BlueDataUtility { @@ -35,19 +36,62 @@ namespace BCG { public static T get(this IBlueRow r, int col_no, T err_value = default(T)) { return get_row_value(r, col_no, err_value); } - public static IEnumerable get_covalues(this IBlueRowCollection rows, string col) { + public static IEnumerable get_column_values(this IBlueRowCollection rows, string col) { List rv = new List(); foreach (IBlueRow r in rows) rv.Add(r.get(col)); return rv; } - public static IEnumerable get_covalues(IEnumerable rows, string col) { + public static IEnumerable get_colvalues(IEnumerable rows, string col) { List rv = new List(); foreach (IBlueRow r in rows) rv.Add(r.get(col)); return rv; } + public static IDictionary> group_by(IBlueRowCollection rows, string gp_col) { + Dictionary> rv = new Dictionary>(); + foreach (IBlueRow r in rows) { + var gp_val = r.get(gp_col); + List gp_val_list = null; + if (!rv.TryGetValue(gp_val, out gp_val_list)) { + gp_val_list = new List(); + rv.Add(gp_val, gp_val_list); + } + gp_val_list.Add(r); + } + return rv.ToDictionary(kv => kv.Key, kv => kv.Value as IEnumerable); + } + + // this function takes in a key value table and converts it into a relational table, converting the key names into columns. + // The values are returned as string values + // doc_col is the column specifyng the doc name (following mongoDB's nomenclature, one set of key value rows ar a 'doc') + public static IBlueRowCollection keyvalue_to_relational(IBlueRowCollection kv_rows, string doc_col, string key_col, string val_col) { + HashSet unique_cols = new HashSet(); + foreach (string col in get_column_values(kv_rows, key_col)) { + unique_cols.Add(col); + } + //unique_cols.Add(doc_col); + var schema_cols = unique_cols.Select(c => Tuple.Create(c, TypeCode.String)).ToList(); + schema_cols.Insert(0, Tuple.Create(doc_col, BlueDataUtility.get_typecode(typeof(T)))); + schema_cols.Insert(0, Tuple.Create("id", TypeCode.Int32)); + var schema = BlueDataUtility.form_schema("relational", schema_cols.ToArray()); + BlueTable rv = new BlueTable(schema); + var gp_map = group_by(kv_rows, doc_col); + foreach (var kv in gp_map) { + MutableBlueRow nr = new MutableBlueRow(schema); + nr.setValue("id", rv.Count); + nr.setValue(doc_col, kv.Key); + foreach (var r in kv.Value) { + var col = r.get(key_col); + var val = r.get(val_col); + nr.setValue(col, val); + } + rv.addRow(nr); + } + return rv; + } + public static IBlueRowCollection where(this IBlueRowCollection rows, Func where_func) { if (rows == null) return null; @@ -179,6 +223,15 @@ namespace BCG { { TypeCode.UInt64, typeof(UInt64) } }; + public static TypeCode get_typecode(Type t) { + foreach (var kv in _typecode_to_type) { + if (kv.Value == t) + return kv.Key; + } + bts.Log.assert_d(false, $"type: {t} not found"); + return TypeCode.Empty; + } + public static IBlueTableSchema form_schema(string table_name, params Tuple[] cols) { List cis = new List(); foreach (Tuple c in cols) diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs index bc5e2e974..c8acd60d4 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Program.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs @@ -59,13 +59,13 @@ void _publish_streams() { CloudStreamManager csm = new CloudStreamManager(); // get it from config Dictionary streams_to_publish = new Dictionary() { - {"bbo","bbo()"}, - {"instruments","instruments()"}, + //{"bbo","bbo()"}, + //{"instruments","instruments()"}, //{"timesales","timesales(10000)"}, - {"books","books()"}, + //{"books","books()"}, {"seconds_1_10","second(1,10)"}, {"literal_420","literal(\"x\", \"420.0\")"}, - //{"mongodb_testcollection", "mongodb(\"mongodb://192.168.1.80:27017\",\"local\",\"test_collection\")"} + {"mongodb_testcollection", "mongodb(\"mongodb://192.168.1.80:27017\",\"local\",\"test_collection\")"} }; // {"bbo_ts","timeSeries(bbo())"} diff --git a/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs b/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs index 4cc816364..f9d2cb31e 100644 --- a/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs +++ b/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs @@ -253,47 +253,6 @@ namespace BCG { public void doWork() { //_dispose_and_reconnect_if_missing_hb(); _check_for_new_connection(); - ///check for heartbeats and let clients know in - ///case of missing hb (connection breaking) - // TimeSpan hb_interval = DateTime.Now - myLastHB; - // bool hb_missed = hb_interval > myConnectionDropThreshold; - // bool conn_disposed = myConnection != null || myConnection.isDisposed; - // if (conn_disposed || hb_missed) - // { - // if (hb_missed) - // { - // // since conn breakage was diagnosed logically, lets physically close the connection - // myConnection.Dispose(); - // string dispose_msg = $"PubSubServerManagedTCPConnection.doWork server: {myServerAddress} Disposing connection conn_disposed: {conn_disposed} hb_missed: {hb_missed}"; - // Console.WriteLine(dispose_msg); - // } - // myConnection = null; // this is used as a marker for a dead connection - // if (myConnCallback != null) - // myConnCallback(myServerAddress.ToString(), false); - // string reconn_msg = $"PubSubServerManagedTCPConnection.doWork disconnecting client from server: {myServerAddress} conn_disposed: {conn_disposed} hb_missed: {hb_missed} " + - // $"because of missing hbs last_hb: {myLastHB}, now: {DateTime.Now} curr_hb_interval: {hb_interval.TotalSeconds} threshold: {myConnectionDropThreshold.TotalSeconds} conn_disposed: {conn_disposed}"; - // ClientToServerManagedTCPConnector.connectTo(myServerAddress, this); - // } - // if (myTempNewConnection != null) - // { - // if (myConnection != null) - // { - // // TODOds - // // log error and close it - // // BlueEventLog.logAssert(myConnection == null, "Connection has to be nulled out and disposed " - // // + " before a new connection is established"); - // } - // Console.WriteLine("PubSubServerManagedTCPConnection hooking new connection"); - // myConnection = myTempNewConnection; - // myTempNewConnection = null; - // myConnection.register(this); - // myConfs.dequeueAll(); - // if (myConnCallback != null) - // myConnCallback(myServerAddress.ToString(), true); - // } - // foreach (Pair p in myConfs.dequeueAll()) - // if (myOnSendConfirmCallback != null) - // myOnSendConfirmCallback(p.first, p.second); } #endregion diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs index ef1f14a19..ac4d7c952 100644 --- a/src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs +++ b/src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs @@ -16,6 +16,7 @@ namespace BCG { // SubMetaData // TransactionMetaData // UnSubMetaData + // public class MetaData : ICloneable { private IDictionary _name_value_map = new Dictionary(); int mySize = -1; diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/SubMetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/SubMetaData.cs index 3b59c69c6..6c09a4c55 100644 --- a/src/DataStreamSuite/PubSub/Client/MetaData/SubMetaData.cs +++ b/src/DataStreamSuite/PubSub/Client/MetaData/SubMetaData.cs @@ -1,10 +1,8 @@ using System; -namespace BCG -{ - [Serializable] - public class SubMetaData - { +namespace BCG { + //[Serializable] + public class SubMetaData { /// /// connection URL on which the connection will receive data /// @@ -13,20 +11,18 @@ namespace BCG /// identification of the client and the connection on which to receive data /// machine; ip; port; clientURL /// - public string subscribingApp; + public readonly string subscribing_app = ClientInfo.thisClient.processString; /// /// tags that are to be subscribed /// public string[] tags; - public SubMetaData() - { } + public SubMetaData() { } - public SubMetaData(string theConnection, params string[] theTags) - { - connection = theConnection; - tags = theTags; - subscribingApp = ClientInfo.thisClient.processString; + public SubMetaData(string the_connection, params string[] the_tags) { + connection = the_connection; + tags = the_tags; + //subscribing_app = ClientInfo.thisClient.processString; } } } diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/UnSubMetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/UnSubMetaData.cs index 2bc7540fe..deb98d8d8 100644 --- a/src/DataStreamSuite/PubSub/Client/MetaData/UnSubMetaData.cs +++ b/src/DataStreamSuite/PubSub/Client/MetaData/UnSubMetaData.cs @@ -1,21 +1,17 @@ using System; -namespace BCG -{ - [Serializable] - public class UnSubMetaData - { +namespace BCG { + //[Serializable] + public class UnSubMetaData { public string connection; - public string[] tags; - public string unsubscribingApp = ClientInfo.thisClient.processString; + public readonly string[] tags; + public readonly string unsubscribing_app = ClientInfo.thisClient.processString; - public UnSubMetaData() - {} + public UnSubMetaData() { } - public UnSubMetaData(string theConnection, params string[] theTags) - { - connection = theConnection; - tags = theTags; + public UnSubMetaData(string the_connection, params string[] the_tags) { + connection = the_connection; + tags = the_tags; } } } diff --git a/src/DataStreamSuite/PubSub/PubSubUpdate.cs b/src/DataStreamSuite/PubSub/PubSubUpdate.cs index 8f987cff3..832af2b82 100644 --- a/src/DataStreamSuite/PubSub/PubSubUpdate.cs +++ b/src/DataStreamSuite/PubSub/PubSubUpdate.cs @@ -4,19 +4,22 @@ using System.IO; namespace BCG { /// - /// this is the data returned from the - /// pubsub server + /// this is the data returned from the pubsub server /// public class PubSubUpdate { - public PubSubUpdate(string tag, string tagType, Stream body) { this.tag = tag; this.body = body; this.tagType = tagType; } + public PubSubUpdate(string tag, string the_tag_type, Stream body) { + this.tag = tag; + this.body = body; + this.tag_type = the_tag_type; + } public string tag; public Stream body; - public string tagType; + public string tag_type; public override string ToString() { - return $"PubSubUpdate tag: {tag} tag_type: {tagType} len: {body.Position}"; + return $"PubSubUpdate tag: {tag} tag_type: {tag_type} len: {body.Position}"; } }; } diff --git a/src/DataStreamSuite/PubSub/Server/PubSubState.cs b/src/DataStreamSuite/PubSub/Server/PubSubState.cs index 382d093cd..d24360efd 100644 --- a/src/DataStreamSuite/PubSub/Server/PubSubState.cs +++ b/src/DataStreamSuite/PubSub/Server/PubSubState.cs @@ -55,7 +55,7 @@ namespace BCG { myTagToPubsubMap[tag].onSub(sub); if (onSingleTagChange != null) - onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribingApp)); + onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribing_app)); } if (onTagMapChanged != null) onTagMapChanged(sub.tags); @@ -68,10 +68,10 @@ namespace BCG { ///and publishers for the tag then remove it. myTagToPubsubMap[tag].onUnsub(unsub); if (onSingleTagChange != null) - onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribingApp)); + onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribing_app)); } else { string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" + - " the unsub call by app {1} is voided", tag, unsub.unsubscribingApp); + " the unsub call by app {1} is voided", tag, unsub.unsubscribing_app); //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning); } } diff --git a/src/DataStreamSuite/PubSub/Server/SingleTagPubSubMap.cs b/src/DataStreamSuite/PubSub/Server/SingleTagPubSubMap.cs index a466fa71b..d9c30ad8d 100644 --- a/src/DataStreamSuite/PubSub/Server/SingleTagPubSubMap.cs +++ b/src/DataStreamSuite/PubSub/Server/SingleTagPubSubMap.cs @@ -1,19 +1,19 @@ using System; using System.Collections.Generic; using System.Text; + using BCG.Utilities; + using mbts; -namespace BCG -{ +namespace BCG { /// /// SingleTagPubSubMap consists of all the state w.r.t to /// publishing and subscription of a single tag. this information /// consists of the publisher, and a map of the subscriber /// connection vs the apps they're coming from /// - internal class SingleTagPubSubMap - { + internal class SingleTagPubSubMap { private string myTag; /// /// map of clientURL to client app (machine;app;pid;clientURL) @@ -29,8 +29,7 @@ namespace BCG /// List> myDeadPubConnToApp = new List>(); - public SingleTagPubSubMap(string tag) - { + public SingleTagPubSubMap(string tag) { myTag = tag; } @@ -38,8 +37,7 @@ namespace BCG /// Returns a list all the active connections that /// have subscribed to this tag /// - public ICollection subscriberConnections - { + public ICollection subscriberConnections { get { return mySubConnToApp.Keys; } } @@ -47,8 +45,7 @@ namespace BCG /// Returns a list of all the applications that have /// subscribed to this tag /// - public ICollection subscripingApps - { + public ICollection subscripingApps { get { return mySubConnToApp.Values; } } @@ -57,8 +54,7 @@ namespace BCG /// connections as key and the application /// as value /// - public Dictionary subscribers - { + public Dictionary subscribers { get { return mySubConnToApp; } } @@ -66,8 +62,7 @@ namespace BCG /// Returns a dictionary for all active publishing /// connection as key and the application as value /// - public Dictionary activePublishers - { + public Dictionary activePublishers { get { return myActivePubConnToApp; } } @@ -75,8 +70,7 @@ namespace BCG /// Returns a list of pair of dead connections and /// theur corresponding applications /// - public ICollection> deadPublishers - { + public ICollection> deadPublishers { get { return myDeadPubConnToApp; } } @@ -87,18 +81,14 @@ namespace BCG /// /// /// - public bool onSub(SubMetaData sub) - { - if (mySubConnToApp.ContainsKey(sub.connection)) - { + public bool onSub(SubMetaData sub) { + if (mySubConnToApp.ContainsKey(sub.connection)) { string warningMsg = String.Format("tag {0} has already been subscribed by app {0}, subscriber {1}" - + "it should unsubscribe before resubscribing", myTag, sub.connection, sub.subscribingApp); + + "it should unsubscribe before resubscribing", myTag, sub.connection, sub.subscribing_app); //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning); return false; - } - else - { - mySubConnToApp.Add(sub.connection, sub.subscribingApp); + } else { + mySubConnToApp.Add(sub.connection, sub.subscribing_app); return true; } } @@ -110,17 +100,13 @@ namespace BCG /// /// /// - public bool onUnsub(UnSubMetaData unsub) - { - if (mySubConnToApp.ContainsKey(unsub.connection)) - { + public bool onUnsub(UnSubMetaData unsub) { + if (mySubConnToApp.ContainsKey(unsub.connection)) { mySubConnToApp.Remove(unsub.connection); return true; - } - else - { + } else { string warningMsg = String.Format("tag {0} has already was either unsubscribed or never subscribed by app {0}, subscriber {1}", - myTag, unsub.connection, unsub.unsubscribingApp); + myTag, unsub.connection, unsub.unsubscribing_app); //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning); return false; } @@ -133,25 +119,20 @@ namespace BCG /// /// /// - public bool onConnectionRemoveIfExists(string connection, out bool wasPublisher, out bool wasSubscriber) - { + public bool onConnectionRemoveIfExists(string connection, out bool wasPublisher, out bool wasSubscriber) { bool rv = false; - if (mySubConnToApp.ContainsKey(connection)) - { + if (mySubConnToApp.ContainsKey(connection)) { mySubConnToApp.Remove(connection); wasSubscriber = true; rv = true; - } - else + } else wasSubscriber = false; - if (myActivePubConnToApp.ContainsKey(connection)) - { + if (myActivePubConnToApp.ContainsKey(connection)) { myDeadPubConnToApp.Add(new Pair(connection, myActivePubConnToApp[connection])); myActivePubConnToApp.Remove(connection); rv = true; wasPublisher = true; - } - else + } else wasPublisher = false; return rv; } @@ -162,14 +143,10 @@ namespace BCG /// /// /// - public void onPub(PubMetaData pub, out bool newPublisher) - { - if (myActivePubConnToApp.ContainsKey(pub.connection)) - { + public void onPub(PubMetaData pub, out bool newPublisher) { + if (myActivePubConnToApp.ContainsKey(pub.connection)) { newPublisher = false; - } - else - { + } else { myActivePubConnToApp.Add(pub.connection, pub.publishingApp); newPublisher = true; } diff --git a/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs b/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs index 32bb09357..e4600a51d 100644 --- a/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs +++ b/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs @@ -6,92 +6,79 @@ using BCG.Utilities; using BCG.Utilities.General; using mbts; using mbts.spark; +using System.Linq; namespace BCG { /// - /// Deals with the subscriptions (and getting data) from one - /// pubsub server. Once again, theres no sharing of conenctions + /// Deals with the subscriptions (and getting data) from one sole + /// pubsub server. Theres no sharing of conenctions /// from this point. Every new instance of SingleServerSubscriberCore /// will open its new dedicated connection to the serverURL that /// its concerned with (passed as a conctructor arg). /// internal interface ISingleServerSubscriberCore : IDisposable { - IList getUpdates(int timeout); + IList get_updates(int timeout); event PubSubConnectionChange onPubSubConnectionChange; void subscribe(params string[] tags); void unsubscribe(params string[] tags); } + internal class SingleServerSubscriberCore : IDisposable, ISingleServerSubscriberCore { - private HighPerformanceTimer myTimer = new HighPerformanceTimer(); - private PubSubServerConnectionFactory myConnectionFactory; - private IPubSubServerConnection myConnection; - private List myTags = new List(); + private readonly IPubSubServerConnection _connection; + private HashSet _tags = new HashSet(); public event PubSubConnectionChange onPubSubConnectionChange; - public SingleServerSubscriberCore(string serverURL, PubSubServerConnectionFactory connectionFactory) { - myConnectionFactory = connectionFactory; - myConnection = myConnectionFactory.getConnection(serverURL); - myConnection.register_connection_callback(new PubSubServerConnectionStatusChange(myHandleServerReboot)); + public SingleServerSubscriberCore(string server_url, PubSubServerConnectionFactory connection_factory) { + _connection = connection_factory.getConnection(server_url); + _connection.register_connection_callback(new PubSubServerConnectionStatusChange(_handle_server_reboot)); } - public IList getUpdates() { - return getUpdates(1); - } - - /// - /// gets as many data items that it can get in - /// timeout milliseconds - /// - /// - /// - public IList getUpdates(int timeout) { - myConnection.doWork(); - List retVal = new List(); - IList dataItems = myConnection.get_updates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue); - foreach (byte[] data in dataItems) { - processMessage(data, retVal); + // gets as many data items that it can get in + // timeout milliseconds + public IList get_updates(int timeout) { + _connection.doWork(); + List rv = new List(); + IList data_items = _connection.get_updates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue); + foreach (byte[] data in data_items) { + process_message(data, ref rv); } - return retVal; + return rv; } - public List getSubscribedTags() { - return myTags; - } - public void subscribe(params string[] tags) { + HashSet this_call_tags = new HashSet(); foreach (string tag in tags) { - if (myTags.Contains(tag)) { - string errMsg = String.Format("tag {0} was already subscribed from server {1}", - tag, myConnection.server_url); - // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning); + if (_tags.Contains(tag)) { + bts.Log.Gen.warn($"SingleServerSubscriberCore.subscribe tag: {tag} has already " + + $"been subscribed to server: {_connection.server_url}"); } else { - myTags.Add(tag); + _tags.Add(tag); + this_call_tags.Add(tag); } } - mySubscribe(tags); + _subscribe(this_call_tags.ToArray()); } public void unsubscribe(params string[] tags) { - List unsubTags = new List(); + List unsub_tags = new List(); foreach (string tag in tags) { - if (myTags.Contains(tag)) { - myTags.Remove(tag); - unsubTags.Add(tag); + if (_tags.Contains(tag)) { + _tags.Remove(tag); + unsub_tags.Add(tag); } else { - string errMsg = String.Format("tag {0} is being unsubscribed from server {1}, eventhough " + - "it was never subscribed", tag, myConnection.server_url); - // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning); + bts.Log.Gen.warn($"tag {tag} is being unsubscribed from server {_connection.server_url}, eventhough " + + "it was never subscribed"); } } - UnSubMetaData unsub = new UnSubMetaData(myConnection.client_url, unsubTags.ToArray()); + UnSubMetaData unsub = new UnSubMetaData(_connection.client_url, unsub_tags.ToArray()); Stream str = new MemoryStream(); new MetaData(unsub).save(str); str.Position = 0; // this is where data is actually sent out - myConnection.send(str); + _connection.send(str); } #region IDisposable Members @@ -99,44 +86,40 @@ namespace BCG { public void Dispose() { ///remove the subscriber //myConnection.send(new PubSubMessage(new MetaData(new RemoveSubscriberMetaData(myConnection.clientURL))).toStream()); - myConnection.Dispose(); + _connection.Dispose(); //clear the list of tags you've subscribed to. This way, if you're ever used again, //you have to send all the subscription metadata out again. - myTags.Clear(); + _tags.Clear(); } #endregion - private void mySubscribe(params string[] tags) { - SubMetaData sub = new SubMetaData(myConnection.client_url, tags); + // The only place where a tag is subsribed from the Pubsub server. the subscriptions are wrapped into SubMetaData, + // which has a to_stream method + private void _subscribe(params string[] tags) { + SubMetaData sub = new SubMetaData(_connection.client_url, tags); Stream str = new MemoryStream(); new MetaData(sub).save(str); str.Position = 0; - myConnection.send(str); + _connection.send(str); } - private void myHandleServerReboot(string serverURL, bool connection) { + private void _handle_server_reboot(string server_url, bool connection) { + bts.Log.Gen.info($"SingleServerSubscriberCore._handle_server_reboot connection: {connection} server_url: {server_url}"); if (connection) { - //ie connection established again, so has to - //resubscribe to all the tags - // BlueEventLog.logEvent("Connection to " + serverURL + " established....resubscribing to all the tags", - // PS3LogLevels.disconnection); - mySubscribe(myTags.ToArray()); - - + _subscribe(_tags.ToArray()); } else { - // BlueEventLog.logEvent("Connection to " + serverURL + " broke....attempting to reconnect...", - // PS3LogLevels.disconnection); + bts.Log.Gen.warn($"SingleServerSubscriberCore._handle_server_reboot connection to {server_url} broke....attempting to reconnect..."); } - //let anyone watching you know that your connection has changed. if (onPubSubConnectionChange != null) - onPubSubConnectionChange(connection, serverURL); + onPubSubConnectionChange(connection, server_url); } - internal static void processMessage(byte[] data, List updates) { - bts.Log.Gen.info($"SingleServerSubscriberCore.processMessage data_len: {data.Length}"); + // processes one message + internal static void process_message(byte[] data, ref List updates) { + bts.Log.Gen.info($"SingleServerSubscriberCore.process_message data_len: {data.Length}"); if (data != null && data.Length > 0) { MemoryStream str = new MemoryStream(data); MetaData md = new MetaData(str); @@ -156,9 +139,8 @@ namespace BCG { if (p.isCompressed) { long compressedSize = bodyStream.Length - bodyStream.Position; if (compressedSize == 0) { - string errorMsg = "Error: tag " + p.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update."; - Console.WriteLine(errorMsg); - //BlueEventLog.logEvent(errorMsg, BlueEventLevel.Error); + string error_msg = "Error: tag " + p.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update."; + bts.Log.Gen.error(error_msg); return; } realBody = StreamDecompressor.decompressStream(bodyStream); @@ -174,8 +156,9 @@ namespace BCG { bodyStream.Read(trConstituent, 0, trConstituent.Length); updates.Add(new PubSubUpdate(pub.tag, pub.type, new MemoryStream(trConstituent))); } - } else { } - //BlueEventLog.logAssert(false, "Unknown message received: " + md.ToString() + " , only Pub and TransactionMetaData are legal"); + } else { + bts.Log.assert_d(false, $"Unknown message received: {md} only Pub and TransactionMetaData are legal"); + } } } } diff --git a/src/DataStreamSuite/PubSub/StaticSubscriberCore.cs b/src/DataStreamSuite/PubSub/StaticSubscriberCore.cs index 1911ae8d3..fdcf2ff2e 100644 --- a/src/DataStreamSuite/PubSub/StaticSubscriberCore.cs +++ b/src/DataStreamSuite/PubSub/StaticSubscriberCore.cs @@ -80,7 +80,7 @@ namespace BCG { int timeShare = timeout / myServerURLToSubCore.Count; List retVal = new List(); foreach (ISingleServerSubscriberCore subCore in myServerURLToSubCore.Values) - retVal.AddRange(subCore.getUpdates(timeShare)); + retVal.AddRange(subCore.get_updates(timeShare)); _log_n_updates(retVal); myTimer.stop(); //if (myTimer.lastIntervalTime > 10) diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs index 82586a5dd..14cea626b 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs @@ -1,26 +1,20 @@ -namespace BCG -{ +namespace BCG { using bts.api.orderman; - enum TSGroupByOperation - { + enum TSGroupByOperation { OHLC = 0, Statistics } [DataStreamUserRespresentation("timeseriesanalysis")] - public class TimeSeriesGroupByStream : ViewStream - { - public TimeSeriesGroupByStream(ViewStream md_str, string timebucket_col, string output_col, string operation) - { + public class TimeSeriesGroupByStream : ViewStream { + public TimeSeriesGroupByStream(ViewStream md_str, string timebucket_col, string output_col, string operation) { ViewStream rs = new NonIncGroupByStream(md_str, new OHLCNonIncGroupByEvaluator(timebucket_col, output_col)); mySetRealSource(rs); } } - public class OHLC - { - public OHLC(double open_args, double high_args, double low_args, double close_args) - { + public class OHLC { + public OHLC(double open_args, double high_args, double low_args, double close_args) { open = open_args; high = high_args; low = low_args; @@ -28,47 +22,36 @@ namespace BCG } public readonly double open, high, low, close; } - public class OHLCNonIncGroupByEvaluator : INonIncGroupByEvaluator - { + public class OHLCNonIncGroupByEvaluator : INonIncGroupByEvaluator { private string _timestamp_col; private string _output_col; private IBlueTableSchema _out_schema; - public OHLCNonIncGroupByEvaluator(string timestamp_col, string output_col) - { + public OHLCNonIncGroupByEvaluator(string timestamp_col, string output_col) { _timestamp_col = timestamp_col; _output_col = output_col; } - string open_col - { - get - { + string open_col { + get { return $"{_output_col}_open"; ; } } - string high_col - { - get - { + string high_col { + get { return $"{_output_col}_high"; ; } } - string low_col - { - get - { + string low_col { + get { return $"{_output_col}_low"; ; } } - string close_col - { - get - { + string close_col { + get { return $"{_output_col}_close"; ; } } - private void _build_schema(IBlueTableSchema input_schema) - { + private void _build_schema(IBlueTableSchema input_schema) { List> new_cols = new List>(); ColumnInfo ci = input_schema[_output_col]; new_cols.Add(Tuple.Create(ci.colType, open_col)); @@ -79,32 +62,25 @@ namespace BCG _out_schema = BlueDataUtility.add_columns(input_schema, new_cols.ToArray()); } - public IBlueRowCollection evaluate(IDictionary> grouped_rows) - { - if (_out_schema == null) - { - if (grouped_rows.Count > 0) - { - foreach (var kv in grouped_rows) - { + public IBlueRowCollection evaluate(IDictionary> grouped_rows) { + if (_out_schema == null) { + if (grouped_rows.Count > 0) { + foreach (var kv in grouped_rows) { var rows = kv.Value; - if (rows != null && rows.Count > 0) - { + if (rows != null && rows.Count > 0) { _build_schema(rows.ElementAt(0).schema); } } } } BlueTable rv = new BlueTable(_out_schema); - foreach (var kv in grouped_rows) - { + foreach (var kv in grouped_rows) { int id = kv.Key; IEnumerable rows = kv.Value; // sort rows on time to compute o,c correctly?? - if (rows != null && rows.Count() > 0) - { + if (rows != null && rows.Count() > 0) { IBlueRow reprow = rows.ElementAt(0); - IEnumerable vals = BlueDataUtility.get_covalues(rows, _output_col); + IEnumerable vals = BlueDataUtility.get_colvalues(rows, _output_col); OHLC ohlc = compute_ohlc(vals); MutableBlueRow nr = new MutableBlueRow(_out_schema, reprow); nr.setValue("id", kv.Key); @@ -118,13 +94,11 @@ namespace BCG return rv; } - public string[] get_group_by_cols() - { + public string[] get_group_by_cols() { return new string[] { _timestamp_col }; } - static OHLC compute_ohlc(IEnumerable prices) - { + static OHLC compute_ohlc(IEnumerable prices) { double o = prices.ElementAt(0); double c = prices.ElementAt(prices.Count() - 1); double l = prices.Min(); @@ -133,28 +107,22 @@ namespace BCG } } [DataStreamUserRespresentation("RSI")] - public class RSIStream : ViewStream - { + public class RSIStream : ViewStream { // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for - public RSIStream(ViewStream md_str, string timebucket_column, string output_col) - { + public RSIStream(ViewStream md_str, string timebucket_column, string output_col) { } } [DataStreamUserRespresentation("Bollinger")] - public class BollingerBandStream : ViewStream - { + public class BollingerBandStream : ViewStream { // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for - public BollingerBandStream(ViewStream md_str, string timebucket_column, string output_col) - { + public BollingerBandStream(ViewStream md_str, string timebucket_column, string output_col) { } } [DataStreamUserRespresentation("OHLC")] - public class OHLCStream : ViewStream - { + public class OHLCStream : ViewStream { // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for - public OHLCStream(ViewStream md_str, string timebucket_column, string output_col) - { + public OHLCStream(ViewStream md_str, string timebucket_column, string output_col) { ViewStream rs = new TimeSeriesGroupByStream(md_str, timebucket_column, output_col, TSGroupByOperation.OHLC.ToString()); mySetRealSource(rs); }