diff --git a/src/DataStreamSuite/DataStream/DataStream.csproj b/src/DataStreamSuite/DataStream/DataStream.csproj
index 960706a61..577b35b5f 100644
--- a/src/DataStreamSuite/DataStream/DataStream.csproj
+++ b/src/DataStreamSuite/DataStream/DataStream.csproj
@@ -15,6 +15,7 @@
+
diff --git a/src/DataStreamSuite/DataStream/Primitives/KeyValue/KeyValueToRelationalStream.cs b/src/DataStreamSuite/DataStream/Primitives/KeyValue/KeyValueToRelationalStream.cs
new file mode 100644
index 000000000..2a2fed4a4
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Primitives/KeyValue/KeyValueToRelationalStream.cs
@@ -0,0 +1,31 @@
+namespace BCG {
+ using System.Collections.Generic;
+
+ public class KeyValueToRelationalStream : ViewStream {
+ private readonly ViewStream _kv_str;
+ private readonly string _doc_id_col, _key_col, _val_col;
+ public KeyValueToRelationalStream(ViewStream kv_str, string doc_id_col, string key_col, string val_col) {
+ _kv_str = kv_str;
+ _doc_id_col = doc_id_col;
+ _key_col = key_col;
+ _val_col = val_col;
+ }
+
+ protected override void myOnFirstSubscriber() {
+ _kv_str.onChange += _on_str_update;
+ }
+
+ protected override void myOnLastSubscriber() {
+ _kv_str.onChange -= _on_str_update;
+ }
+
+ private void _on_str_update(object dp_obj) {
+ DataPacket dp = dp_obj as DataPacket;
+ var rows = dp.newRows;
+ var rv = BlueDataUtility.keyvalue_to_relational(rows, _doc_id_col, _key_col, _val_col);
+ var dp_out = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket, true);
+ myFireEvent(dp_out);
+ }
+
+ }
+}
\ No newline at end of file
diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/WebScrapingStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/WebScrapingStream.cs
new file mode 100644
index 000000000..9f70da850
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Primitives/Misc/WebScrapingStream.cs
@@ -0,0 +1,121 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Security.Cryptography.X509Certificates;
+
+using BCG;
+using BCG.Utilities.General;
+
+using HtmlAgilityPack;
+
+namespace BCG {
+ class Program {
+ static void Main() {
+ string url = "http://quotes.toscrape.com";
+ List quotes = ScrapeQuotes(url);
+
+ Console.WriteLine("Quotes from http://quotes.toscrape.com:");
+ foreach (var quote in quotes) {
+ Console.WriteLine(quote);
+ }
+ }
+
+ static List ScrapeQuotes(string url) {
+ List quotes = new List();
+
+ // Create a new HtmlWeb instance
+ HtmlWeb web = new HtmlWeb();
+
+ try {
+ // Load the HTML document from the specified URL
+ HtmlDocument document = web.Load(url);
+
+ // Extract quotes from the HTML document
+ var quoteNodes = document.DocumentNode.SelectNodes("//span[@class='text']");
+ if (quoteNodes != null) {
+ foreach (var quoteNode in quoteNodes) {
+ string quote = quoteNode.InnerText.Trim();
+ quotes.Add(quote);
+ }
+ }
+ } catch (Exception ex) {
+ Console.WriteLine($"Error: {ex.Message}");
+ }
+
+ return quotes;
+ }
+ }
+
+ public class WebScapperStream : ViewStream, IWorker {
+ private readonly string _url;
+ private readonly string _xpath;
+ private readonly ViewStreamDatabase _vsd;
+ private IDictionary _line_id_map = new Dictionary();
+ private readonly IBlueTableSchema _schema;
+
+ public WebScapperStream()
+ :
+ this("http://quotes.toscrape.com", "//span[@class='text']", ViewStreamDatabase.instance) { }
+ public WebScapperStream(string url, string xpath)
+ : this(url, xpath, ViewStreamDatabase.instance) { }
+ public WebScapperStream(string url, string xpath, ViewStreamDatabase vsd) {
+ bts.Log.Gen.info($"WebScapperStream cons url: {url} xpath: {xpath}");
+ _schema = BlueDataUtility.form_schema($"{url}_{xpath}",
+ Tuple.Create("id", TypeCode.Int32),
+ Tuple.Create("url", TypeCode.UInt32),
+ Tuple.Create("line", TypeCode.String));
+ _url = url;
+ _xpath = xpath;
+ _vsd = vsd;
+ }
+
+ protected override void myOnFirstSubscriber() {
+ MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+ }
+
+ protected override void myOnLastSubscriber() {
+ MasterWorker.getInstance(false).removeWorker(this);
+ }
+
+ public void doWork() {
+ var lines = scrape_lines(_url, _xpath);
+ Func get_id = (line) => {
+ int rv = 0;
+ if (!_line_id_map.TryGetValue(line, out rv))
+ _line_id_map.Add(line, _line_id_map.Count);
+ return rv;
+ };
+ BlueTable nrows = new BlueTable(_schema);
+ foreach (var l in lines) {
+ bts.Log.Gen.info($"line: {l}");
+ var id = get_id(l);
+ BlueRow nr = new BlueRow(_schema, get_id(l), _url, l);
+ nrows.addRow(nr);
+ }
+ var dp = BlueDataUtility.build_dp_inc(nrows, myLastValue as DataPacket, true);
+ myFireEvent(dp);
+ }
+
+ static List scrape_lines(string url, string xpath) {
+ List quotes = new List();
+ // Create a new HtmlWeb instance
+ HtmlWeb web = new HtmlWeb();
+ try {
+ // Load the HTML document from the specified URL
+ HtmlDocument document = web.Load(url);
+ // Extract quotes from the HTML document
+ var quote_nodes = document.DocumentNode.SelectNodes(xpath);
+ if (quote_nodes != null) {
+ foreach (var qn in quote_nodes) {
+ string quote = qn.InnerText.Trim();
+ quotes.Add(quote);
+ }
+ }
+ } catch (Exception ex) {
+ Console.WriteLine($"Error: {ex.Message}");
+ }
+
+ return quotes;
+ }
+ }
+}
diff --git a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs
index bb2026f94..52015e568 100644
--- a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs
@@ -9,42 +9,9 @@ using System.Runtime.CompilerServices;
using mbts.spark;
-class Program {
- static void Main() {
- // Set the connection string and database name
- string connectionString = "mongodb://localhost:27017";
- string databaseName = "your_database_name";
- string collectionName = "your_collection_name";
-
- // Create a MongoClient
- MongoClient client = new MongoClient(connectionString);
-
- // Access the database
- IMongoDatabase database = client.GetDatabase(databaseName);
-
- // Access the collection
- IMongoCollection collection = database.GetCollection(collectionName);
-
- // Fetch all documents from the collection
- var documents = collection.Find(Builders.Filter.Empty).ToCursor();
-
- // Iterate through the documents and print values
- foreach (var document in documents.ToEnumerable()) {
- // Assuming your document structure has fields named "Field1", "Field2", etc.
- Console.WriteLine("Field1: " + document["Field1"]);
- Console.WriteLine("Field2: " + document["Field2"]);
- // Add more fields as needed
-
- // Print a separator between documents
- Console.WriteLine(new string('-', 20));
- }
- }
-}
-
-
namespace BCG {
- [DataStreamUserRespresentation("mongodb")]
- public class MongoDBStream : ViewStream, IWorker {
+ [DataStreamUserRespresentation("mongodb_sql")]
+ public class MongoDBSQLStream : ViewStream, IWorker {
private ViewStreamDatabase _vsd;
private readonly string _connection_string, _db_name, _collection_name;
private MongoClient _client;
@@ -55,11 +22,11 @@ namespace BCG {
private IDictionary _id_map = new Dictionary();
// mongodb://192.168.1.80:27017
- public MongoDBStream(string connection_string, string db_name, string collection_name)
+ public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
}
- public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+ public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
$"db_name: {db_name}");
_connection_string = connection_string;
@@ -139,4 +106,107 @@ namespace BCG {
}
}
+ [DataStreamUserRespresentation("mongodb")]
+ public class MongoDBStream : ViewStream, IWorker {
+ private ViewStreamDatabase _vsd;
+ private readonly string _connection_string, _db_name, _collection_name;
+ private MongoClient _client;
+ private BlueTable _table;
+
+ // Access the database
+ private IMongoDatabase _database;
+ private IDictionary _id_map = new Dictionary();
+ // mongodb://192.168.1.80:27017
+
+ public MongoDBStream(string connection_string, string db_name, string collection_name)
+ : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+
+ }
+ public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+ bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+ $"db_name: {db_name}");
+ _connection_string = connection_string;
+ _db_name = db_name;
+ _collection_name = collection_name;
+ _vsd = vsd;
+ _client = new MongoClient(_connection_string);
+ _database = _client.GetDatabase(_db_name);
+ }
+
+ protected override void myOnFirstSubscriber() {
+ MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+ }
+
+ protected override void myOnLastSubscriber() {
+ MasterWorker.getInstance(false).removeWorker(this);
+ }
+
+ public void doWork() {
+ _read_mogodb();
+ }
+
+ private void _read_mogodb() {
+ HighPerformanceTimer timer = new HighPerformanceTimer();
+ timer.start();
+ // Access the collection
+ IMongoCollection collection = _database.GetCollection(name: _collection_name);
+
+ // Fetch all documents from the collection
+ var documents = collection.Find(Builders.Filter.Empty).ToCursor();
+ _table = _table == null ? null : new BlueTable(_table.schema);
+ // Iterate through the documents and print values
+ foreach (var document in documents.ToEnumerable()) {
+ bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+ if (_table == null) {
+ _init_table(document);
+ }
+ _add_doc(document);
+ }
+ timer.stop();
+ bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
+ timer.start();
+ var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+ timer.stop();
+ bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
+ if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+ return;
+ myFireEvent(dp);
+ }
+
+ private void _init_table(BsonDocument doc_row) {
+ bts.Log.assert(_table == null);
+ var schema = BlueDataUtility.form_schema($"{_database}_{_table}",
+ Tuple.Create("id", TypeCode.Int32),
+ Tuple.Create("doc_id", TypeCode.String),
+ Tuple.Create("key", TypeCode.String),
+ Tuple.Create("value", TypeCode.String));
+ _table = new BlueTable(schema);
+ }
+
+ private void _add_doc(BsonDocument doc) {
+ var doc_id_bson = doc["_id"];
+ var doc_id = doc_id_bson.ToString();
+ Func get_id = (string doc_id, string key) => {
+ string hash_key = $"{doc_id}_{key}";
+ int id = 0;
+ if (!_id_map.TryGetValue(hash_key, out id)) {
+ id = _id_map.Count;
+ _id_map.Add(hash_key, id);
+ }
+ return id;
+ };
+ foreach (var n in doc.Names) {
+ string key = n;
+ var id = get_id(doc_id, key);
+ MutableBlueRow nr = new MutableBlueRow(_table.schema);
+ var val = doc[n].ToString();
+ nr.setValue("id", id);
+ nr.setValue("doc_id", doc_id);
+ nr.setValue("key", key);
+ nr.setValue("value", val);
+ _table.addRow(nr);
+ }
+ }
+ }
+
}
\ No newline at end of file
diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs
index 10f5b3ba2..a29191e2a 100644
--- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs
+++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs
@@ -13,6 +13,7 @@ using System.Linq.Expressions;
//using mbts.spark;
using mbts;
using BCG.Extensions;
+using System.Runtime.CompilerServices;
namespace BCG {
public static class BlueDataUtility {
@@ -35,19 +36,62 @@ namespace BCG {
public static T get(this IBlueRow r, int col_no, T err_value = default(T)) {
return get_row_value(r, col_no, err_value);
}
- public static IEnumerable get_covalues(this IBlueRowCollection rows, string col) {
+ public static IEnumerable get_column_values(this IBlueRowCollection rows, string col) {
List rv = new List();
foreach (IBlueRow r in rows)
rv.Add(r.get(col));
return rv;
}
- public static IEnumerable get_covalues(IEnumerable rows, string col) {
+ public static IEnumerable get_colvalues(IEnumerable rows, string col) {
List rv = new List();
foreach (IBlueRow r in rows)
rv.Add(r.get(col));
return rv;
}
+ public static IDictionary> group_by(IBlueRowCollection rows, string gp_col) {
+ Dictionary> rv = new Dictionary>();
+ foreach (IBlueRow r in rows) {
+ var gp_val = r.get(gp_col);
+ List gp_val_list = null;
+ if (!rv.TryGetValue(gp_val, out gp_val_list)) {
+ gp_val_list = new List();
+ rv.Add(gp_val, gp_val_list);
+ }
+ gp_val_list.Add(r);
+ }
+ return rv.ToDictionary(kv => kv.Key, kv => kv.Value as IEnumerable);
+ }
+
+ // this function takes in a key value table and converts it into a relational table, converting the key names into columns.
+ // The values are returned as string values
+ // doc_col is the column specifyng the doc name (following mongoDB's nomenclature, one set of key value rows ar a 'doc')
+ public static IBlueRowCollection keyvalue_to_relational(IBlueRowCollection kv_rows, string doc_col, string key_col, string val_col) {
+ HashSet unique_cols = new HashSet();
+ foreach (string col in get_column_values(kv_rows, key_col)) {
+ unique_cols.Add(col);
+ }
+ //unique_cols.Add(doc_col);
+ var schema_cols = unique_cols.Select(c => Tuple.Create(c, TypeCode.String)).ToList();
+ schema_cols.Insert(0, Tuple.Create(doc_col, BlueDataUtility.get_typecode(typeof(T))));
+ schema_cols.Insert(0, Tuple.Create("id", TypeCode.Int32));
+ var schema = BlueDataUtility.form_schema("relational", schema_cols.ToArray());
+ BlueTable rv = new BlueTable(schema);
+ var gp_map = group_by(kv_rows, doc_col);
+ foreach (var kv in gp_map) {
+ MutableBlueRow nr = new MutableBlueRow(schema);
+ nr.setValue("id", rv.Count);
+ nr.setValue(doc_col, kv.Key);
+ foreach (var r in kv.Value) {
+ var col = r.get(key_col);
+ var val = r.get(val_col);
+ nr.setValue(col, val);
+ }
+ rv.addRow(nr);
+ }
+ return rv;
+ }
+
public static IBlueRowCollection where(this IBlueRowCollection rows, Func where_func) {
if (rows == null)
return null;
@@ -179,6 +223,15 @@ namespace BCG {
{ TypeCode.UInt64, typeof(UInt64) }
};
+ public static TypeCode get_typecode(Type t) {
+ foreach (var kv in _typecode_to_type) {
+ if (kv.Value == t)
+ return kv.Key;
+ }
+ bts.Log.assert_d(false, $"type: {t} not found");
+ return TypeCode.Empty;
+ }
+
public static IBlueTableSchema form_schema(string table_name, params Tuple[] cols) {
List cis = new List();
foreach (Tuple c in cols)
diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs
index bc5e2e974..c8acd60d4 100644
--- a/src/DataStreamSuite/DataStreamPublisher/Program.cs
+++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs
@@ -59,13 +59,13 @@ void _publish_streams() {
CloudStreamManager csm = new CloudStreamManager();
// get it from config
Dictionary streams_to_publish = new Dictionary() {
- {"bbo","bbo()"},
- {"instruments","instruments()"},
+ //{"bbo","bbo()"},
+ //{"instruments","instruments()"},
//{"timesales","timesales(10000)"},
- {"books","books()"},
+ //{"books","books()"},
{"seconds_1_10","second(1,10)"},
{"literal_420","literal(\"x\", \"420.0\")"},
- //{"mongodb_testcollection", "mongodb(\"mongodb://192.168.1.80:27017\",\"local\",\"test_collection\")"}
+ {"mongodb_testcollection", "mongodb(\"mongodb://192.168.1.80:27017\",\"local\",\"test_collection\")"}
};
// {"bbo_ts","timeSeries(bbo())"}
diff --git a/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs b/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs
index 4cc816364..f9d2cb31e 100644
--- a/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs
+++ b/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs
@@ -253,47 +253,6 @@ namespace BCG {
public void doWork() {
//_dispose_and_reconnect_if_missing_hb();
_check_for_new_connection();
- ///check for heartbeats and let clients know in
- ///case of missing hb (connection breaking)
- // TimeSpan hb_interval = DateTime.Now - myLastHB;
- // bool hb_missed = hb_interval > myConnectionDropThreshold;
- // bool conn_disposed = myConnection != null || myConnection.isDisposed;
- // if (conn_disposed || hb_missed)
- // {
- // if (hb_missed)
- // {
- // // since conn breakage was diagnosed logically, lets physically close the connection
- // myConnection.Dispose();
- // string dispose_msg = $"PubSubServerManagedTCPConnection.doWork server: {myServerAddress} Disposing connection conn_disposed: {conn_disposed} hb_missed: {hb_missed}";
- // Console.WriteLine(dispose_msg);
- // }
- // myConnection = null; // this is used as a marker for a dead connection
- // if (myConnCallback != null)
- // myConnCallback(myServerAddress.ToString(), false);
- // string reconn_msg = $"PubSubServerManagedTCPConnection.doWork disconnecting client from server: {myServerAddress} conn_disposed: {conn_disposed} hb_missed: {hb_missed} " +
- // $"because of missing hbs last_hb: {myLastHB}, now: {DateTime.Now} curr_hb_interval: {hb_interval.TotalSeconds} threshold: {myConnectionDropThreshold.TotalSeconds} conn_disposed: {conn_disposed}";
- // ClientToServerManagedTCPConnector.connectTo(myServerAddress, this);
- // }
- // if (myTempNewConnection != null)
- // {
- // if (myConnection != null)
- // {
- // // TODOds
- // // log error and close it
- // // BlueEventLog.logAssert(myConnection == null, "Connection has to be nulled out and disposed "
- // // + " before a new connection is established");
- // }
- // Console.WriteLine("PubSubServerManagedTCPConnection hooking new connection");
- // myConnection = myTempNewConnection;
- // myTempNewConnection = null;
- // myConnection.register(this);
- // myConfs.dequeueAll();
- // if (myConnCallback != null)
- // myConnCallback(myServerAddress.ToString(), true);
- // }
- // foreach (Pair p in myConfs.dequeueAll())
- // if (myOnSendConfirmCallback != null)
- // myOnSendConfirmCallback(p.first, p.second);
}
#endregion
diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs
index ef1f14a19..ac4d7c952 100644
--- a/src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs
+++ b/src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs
@@ -16,6 +16,7 @@ namespace BCG {
// SubMetaData
// TransactionMetaData
// UnSubMetaData
+ //
public class MetaData : ICloneable {
private IDictionary _name_value_map = new Dictionary();
int mySize = -1;
diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/SubMetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/SubMetaData.cs
index 3b59c69c6..6c09a4c55 100644
--- a/src/DataStreamSuite/PubSub/Client/MetaData/SubMetaData.cs
+++ b/src/DataStreamSuite/PubSub/Client/MetaData/SubMetaData.cs
@@ -1,10 +1,8 @@
using System;
-namespace BCG
-{
- [Serializable]
- public class SubMetaData
- {
+namespace BCG {
+ //[Serializable]
+ public class SubMetaData {
///
/// connection URL on which the connection will receive data
///
@@ -13,20 +11,18 @@ namespace BCG
/// identification of the client and the connection on which to receive data
/// machine; ip; port; clientURL
///
- public string subscribingApp;
+ public readonly string subscribing_app = ClientInfo.thisClient.processString;
///
/// tags that are to be subscribed
///
public string[] tags;
- public SubMetaData()
- { }
+ public SubMetaData() { }
- public SubMetaData(string theConnection, params string[] theTags)
- {
- connection = theConnection;
- tags = theTags;
- subscribingApp = ClientInfo.thisClient.processString;
+ public SubMetaData(string the_connection, params string[] the_tags) {
+ connection = the_connection;
+ tags = the_tags;
+ //subscribing_app = ClientInfo.thisClient.processString;
}
}
}
diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/UnSubMetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/UnSubMetaData.cs
index 2bc7540fe..deb98d8d8 100644
--- a/src/DataStreamSuite/PubSub/Client/MetaData/UnSubMetaData.cs
+++ b/src/DataStreamSuite/PubSub/Client/MetaData/UnSubMetaData.cs
@@ -1,21 +1,17 @@
using System;
-namespace BCG
-{
- [Serializable]
- public class UnSubMetaData
- {
+namespace BCG {
+ //[Serializable]
+ public class UnSubMetaData {
public string connection;
- public string[] tags;
- public string unsubscribingApp = ClientInfo.thisClient.processString;
+ public readonly string[] tags;
+ public readonly string unsubscribing_app = ClientInfo.thisClient.processString;
- public UnSubMetaData()
- {}
+ public UnSubMetaData() { }
- public UnSubMetaData(string theConnection, params string[] theTags)
- {
- connection = theConnection;
- tags = theTags;
+ public UnSubMetaData(string the_connection, params string[] the_tags) {
+ connection = the_connection;
+ tags = the_tags;
}
}
}
diff --git a/src/DataStreamSuite/PubSub/PubSubUpdate.cs b/src/DataStreamSuite/PubSub/PubSubUpdate.cs
index 8f987cff3..832af2b82 100644
--- a/src/DataStreamSuite/PubSub/PubSubUpdate.cs
+++ b/src/DataStreamSuite/PubSub/PubSubUpdate.cs
@@ -4,19 +4,22 @@ using System.IO;
namespace BCG {
///
- /// this is the data returned from the
- /// pubsub server
+ /// this is the data returned from the pubsub server
///
public class PubSubUpdate {
- public PubSubUpdate(string tag, string tagType, Stream body) { this.tag = tag; this.body = body; this.tagType = tagType; }
+ public PubSubUpdate(string tag, string the_tag_type, Stream body) {
+ this.tag = tag;
+ this.body = body;
+ this.tag_type = the_tag_type;
+ }
public string tag;
public Stream body;
- public string tagType;
+ public string tag_type;
public override string ToString() {
- return $"PubSubUpdate tag: {tag} tag_type: {tagType} len: {body.Position}";
+ return $"PubSubUpdate tag: {tag} tag_type: {tag_type} len: {body.Position}";
}
};
}
diff --git a/src/DataStreamSuite/PubSub/Server/PubSubState.cs b/src/DataStreamSuite/PubSub/Server/PubSubState.cs
index 382d093cd..d24360efd 100644
--- a/src/DataStreamSuite/PubSub/Server/PubSubState.cs
+++ b/src/DataStreamSuite/PubSub/Server/PubSubState.cs
@@ -55,7 +55,7 @@ namespace BCG {
myTagToPubsubMap[tag].onSub(sub);
if (onSingleTagChange != null)
- onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribingApp));
+ onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribing_app));
}
if (onTagMapChanged != null)
onTagMapChanged(sub.tags);
@@ -68,10 +68,10 @@ namespace BCG {
///and publishers for the tag then remove it.
myTagToPubsubMap[tag].onUnsub(unsub);
if (onSingleTagChange != null)
- onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribingApp));
+ onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribing_app));
} else {
string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" +
- " the unsub call by app {1} is voided", tag, unsub.unsubscribingApp);
+ " the unsub call by app {1} is voided", tag, unsub.unsubscribing_app);
//BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
}
}
diff --git a/src/DataStreamSuite/PubSub/Server/SingleTagPubSubMap.cs b/src/DataStreamSuite/PubSub/Server/SingleTagPubSubMap.cs
index a466fa71b..d9c30ad8d 100644
--- a/src/DataStreamSuite/PubSub/Server/SingleTagPubSubMap.cs
+++ b/src/DataStreamSuite/PubSub/Server/SingleTagPubSubMap.cs
@@ -1,19 +1,19 @@
using System;
using System.Collections.Generic;
using System.Text;
+
using BCG.Utilities;
+
using mbts;
-namespace BCG
-{
+namespace BCG {
///
/// SingleTagPubSubMap consists of all the state w.r.t to
/// publishing and subscription of a single tag. this information
/// consists of the publisher, and a map of the subscriber
/// connection vs the apps they're coming from
///
- internal class SingleTagPubSubMap
- {
+ internal class SingleTagPubSubMap {
private string myTag;
///
/// map of clientURL to client app (machine;app;pid;clientURL)
@@ -29,8 +29,7 @@ namespace BCG
///
List> myDeadPubConnToApp = new List>();
- public SingleTagPubSubMap(string tag)
- {
+ public SingleTagPubSubMap(string tag) {
myTag = tag;
}
@@ -38,8 +37,7 @@ namespace BCG
/// Returns a list all the active connections that
/// have subscribed to this tag
///
- public ICollection subscriberConnections
- {
+ public ICollection subscriberConnections {
get { return mySubConnToApp.Keys; }
}
@@ -47,8 +45,7 @@ namespace BCG
/// Returns a list of all the applications that have
/// subscribed to this tag
///
- public ICollection subscripingApps
- {
+ public ICollection subscripingApps {
get { return mySubConnToApp.Values; }
}
@@ -57,8 +54,7 @@ namespace BCG
/// connections as key and the application
/// as value
///
- public Dictionary subscribers
- {
+ public Dictionary subscribers {
get { return mySubConnToApp; }
}
@@ -66,8 +62,7 @@ namespace BCG
/// Returns a dictionary for all active publishing
/// connection as key and the application as value
///
- public Dictionary activePublishers
- {
+ public Dictionary activePublishers {
get { return myActivePubConnToApp; }
}
@@ -75,8 +70,7 @@ namespace BCG
/// Returns a list of pair of dead connections and
/// theur corresponding applications
///
- public ICollection> deadPublishers
- {
+ public ICollection> deadPublishers {
get { return myDeadPubConnToApp; }
}
@@ -87,18 +81,14 @@ namespace BCG
///
///
///
- public bool onSub(SubMetaData sub)
- {
- if (mySubConnToApp.ContainsKey(sub.connection))
- {
+ public bool onSub(SubMetaData sub) {
+ if (mySubConnToApp.ContainsKey(sub.connection)) {
string warningMsg = String.Format("tag {0} has already been subscribed by app {0}, subscriber {1}"
- + "it should unsubscribe before resubscribing", myTag, sub.connection, sub.subscribingApp);
+ + "it should unsubscribe before resubscribing", myTag, sub.connection, sub.subscribing_app);
//BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
return false;
- }
- else
- {
- mySubConnToApp.Add(sub.connection, sub.subscribingApp);
+ } else {
+ mySubConnToApp.Add(sub.connection, sub.subscribing_app);
return true;
}
}
@@ -110,17 +100,13 @@ namespace BCG
///
///
///
- public bool onUnsub(UnSubMetaData unsub)
- {
- if (mySubConnToApp.ContainsKey(unsub.connection))
- {
+ public bool onUnsub(UnSubMetaData unsub) {
+ if (mySubConnToApp.ContainsKey(unsub.connection)) {
mySubConnToApp.Remove(unsub.connection);
return true;
- }
- else
- {
+ } else {
string warningMsg = String.Format("tag {0} has already was either unsubscribed or never subscribed by app {0}, subscriber {1}",
- myTag, unsub.connection, unsub.unsubscribingApp);
+ myTag, unsub.connection, unsub.unsubscribing_app);
//BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
return false;
}
@@ -133,25 +119,20 @@ namespace BCG
///
///
///
- public bool onConnectionRemoveIfExists(string connection, out bool wasPublisher, out bool wasSubscriber)
- {
+ public bool onConnectionRemoveIfExists(string connection, out bool wasPublisher, out bool wasSubscriber) {
bool rv = false;
- if (mySubConnToApp.ContainsKey(connection))
- {
+ if (mySubConnToApp.ContainsKey(connection)) {
mySubConnToApp.Remove(connection);
wasSubscriber = true;
rv = true;
- }
- else
+ } else
wasSubscriber = false;
- if (myActivePubConnToApp.ContainsKey(connection))
- {
+ if (myActivePubConnToApp.ContainsKey(connection)) {
myDeadPubConnToApp.Add(new Pair(connection, myActivePubConnToApp[connection]));
myActivePubConnToApp.Remove(connection);
rv = true;
wasPublisher = true;
- }
- else
+ } else
wasPublisher = false;
return rv;
}
@@ -162,14 +143,10 @@ namespace BCG
///
///
///
- public void onPub(PubMetaData pub, out bool newPublisher)
- {
- if (myActivePubConnToApp.ContainsKey(pub.connection))
- {
+ public void onPub(PubMetaData pub, out bool newPublisher) {
+ if (myActivePubConnToApp.ContainsKey(pub.connection)) {
newPublisher = false;
- }
- else
- {
+ } else {
myActivePubConnToApp.Add(pub.connection, pub.publishingApp);
newPublisher = true;
}
diff --git a/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs b/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs
index 32bb09357..e4600a51d 100644
--- a/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs
+++ b/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs
@@ -6,92 +6,79 @@ using BCG.Utilities;
using BCG.Utilities.General;
using mbts;
using mbts.spark;
+using System.Linq;
namespace BCG {
///
- /// Deals with the subscriptions (and getting data) from one
- /// pubsub server. Once again, theres no sharing of conenctions
+ /// Deals with the subscriptions (and getting data) from one sole
+ /// pubsub server. Theres no sharing of conenctions
/// from this point. Every new instance of SingleServerSubscriberCore
/// will open its new dedicated connection to the serverURL that
/// its concerned with (passed as a conctructor arg).
///
internal interface ISingleServerSubscriberCore : IDisposable {
- IList getUpdates(int timeout);
+ IList get_updates(int timeout);
event PubSubConnectionChange onPubSubConnectionChange;
void subscribe(params string[] tags);
void unsubscribe(params string[] tags);
}
+
internal class SingleServerSubscriberCore : IDisposable, ISingleServerSubscriberCore {
- private HighPerformanceTimer myTimer = new HighPerformanceTimer();
- private PubSubServerConnectionFactory myConnectionFactory;
- private IPubSubServerConnection myConnection;
- private List myTags = new List();
+ private readonly IPubSubServerConnection _connection;
+ private HashSet _tags = new HashSet();
public event PubSubConnectionChange onPubSubConnectionChange;
- public SingleServerSubscriberCore(string serverURL, PubSubServerConnectionFactory connectionFactory) {
- myConnectionFactory = connectionFactory;
- myConnection = myConnectionFactory.getConnection(serverURL);
- myConnection.register_connection_callback(new PubSubServerConnectionStatusChange(myHandleServerReboot));
+ public SingleServerSubscriberCore(string server_url, PubSubServerConnectionFactory connection_factory) {
+ _connection = connection_factory.getConnection(server_url);
+ _connection.register_connection_callback(new PubSubServerConnectionStatusChange(_handle_server_reboot));
}
- public IList getUpdates() {
- return getUpdates(1);
- }
-
- ///
- /// gets as many data items that it can get in
- /// timeout milliseconds
- ///
- ///
- ///
- public IList getUpdates(int timeout) {
- myConnection.doWork();
- List retVal = new List();
- IList dataItems = myConnection.get_updates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue);
- foreach (byte[] data in dataItems) {
- processMessage(data, retVal);
+ // gets as many data items that it can get in
+ // timeout milliseconds
+ public IList get_updates(int timeout) {
+ _connection.doWork();
+ List rv = new List();
+ IList data_items = _connection.get_updates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue);
+ foreach (byte[] data in data_items) {
+ process_message(data, ref rv);
}
- return retVal;
+ return rv;
}
- public List getSubscribedTags() {
- return myTags;
- }
-
public void subscribe(params string[] tags) {
+ HashSet this_call_tags = new HashSet();
foreach (string tag in tags) {
- if (myTags.Contains(tag)) {
- string errMsg = String.Format("tag {0} was already subscribed from server {1}",
- tag, myConnection.server_url);
- // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning);
+ if (_tags.Contains(tag)) {
+ bts.Log.Gen.warn($"SingleServerSubscriberCore.subscribe tag: {tag} has already " +
+ $"been subscribed to server: {_connection.server_url}");
} else {
- myTags.Add(tag);
+ _tags.Add(tag);
+ this_call_tags.Add(tag);
}
}
- mySubscribe(tags);
+ _subscribe(this_call_tags.ToArray());
}
public void unsubscribe(params string[] tags) {
- List unsubTags = new List();
+ List unsub_tags = new List();
foreach (string tag in tags) {
- if (myTags.Contains(tag)) {
- myTags.Remove(tag);
- unsubTags.Add(tag);
+ if (_tags.Contains(tag)) {
+ _tags.Remove(tag);
+ unsub_tags.Add(tag);
} else {
- string errMsg = String.Format("tag {0} is being unsubscribed from server {1}, eventhough " +
- "it was never subscribed", tag, myConnection.server_url);
- // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning);
+ bts.Log.Gen.warn($"tag {tag} is being unsubscribed from server {_connection.server_url}, eventhough " +
+ "it was never subscribed");
}
}
- UnSubMetaData unsub = new UnSubMetaData(myConnection.client_url, unsubTags.ToArray());
+ UnSubMetaData unsub = new UnSubMetaData(_connection.client_url, unsub_tags.ToArray());
Stream str = new MemoryStream();
new MetaData(unsub).save(str);
str.Position = 0;
// this is where data is actually sent out
- myConnection.send(str);
+ _connection.send(str);
}
#region IDisposable Members
@@ -99,44 +86,40 @@ namespace BCG {
public void Dispose() {
///remove the subscriber
//myConnection.send(new PubSubMessage(new MetaData(new RemoveSubscriberMetaData(myConnection.clientURL))).toStream());
- myConnection.Dispose();
+ _connection.Dispose();
//clear the list of tags you've subscribed to. This way, if you're ever used again,
//you have to send all the subscription metadata out again.
- myTags.Clear();
+ _tags.Clear();
}
#endregion
- private void mySubscribe(params string[] tags) {
- SubMetaData sub = new SubMetaData(myConnection.client_url, tags);
+ // The only place where a tag is subsribed from the Pubsub server. the subscriptions are wrapped into SubMetaData,
+ // which has a to_stream method
+ private void _subscribe(params string[] tags) {
+ SubMetaData sub = new SubMetaData(_connection.client_url, tags);
Stream str = new MemoryStream();
new MetaData(sub).save(str);
str.Position = 0;
- myConnection.send(str);
+ _connection.send(str);
}
- private void myHandleServerReboot(string serverURL, bool connection) {
+ private void _handle_server_reboot(string server_url, bool connection) {
+ bts.Log.Gen.info($"SingleServerSubscriberCore._handle_server_reboot connection: {connection} server_url: {server_url}");
if (connection) {
- //ie connection established again, so has to
- //resubscribe to all the tags
- // BlueEventLog.logEvent("Connection to " + serverURL + " established....resubscribing to all the tags",
- // PS3LogLevels.disconnection);
- mySubscribe(myTags.ToArray());
-
-
+ _subscribe(_tags.ToArray());
} else {
- // BlueEventLog.logEvent("Connection to " + serverURL + " broke....attempting to reconnect...",
- // PS3LogLevels.disconnection);
+ bts.Log.Gen.warn($"SingleServerSubscriberCore._handle_server_reboot connection to {server_url} broke....attempting to reconnect...");
}
-
//let anyone watching you know that your connection has changed.
if (onPubSubConnectionChange != null)
- onPubSubConnectionChange(connection, serverURL);
+ onPubSubConnectionChange(connection, server_url);
}
- internal static void processMessage(byte[] data, List updates) {
- bts.Log.Gen.info($"SingleServerSubscriberCore.processMessage data_len: {data.Length}");
+ // processes one message
+ internal static void process_message(byte[] data, ref List updates) {
+ bts.Log.Gen.info($"SingleServerSubscriberCore.process_message data_len: {data.Length}");
if (data != null && data.Length > 0) {
MemoryStream str = new MemoryStream(data);
MetaData md = new MetaData(str);
@@ -156,9 +139,8 @@ namespace BCG {
if (p.isCompressed) {
long compressedSize = bodyStream.Length - bodyStream.Position;
if (compressedSize == 0) {
- string errorMsg = "Error: tag " + p.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update.";
- Console.WriteLine(errorMsg);
- //BlueEventLog.logEvent(errorMsg, BlueEventLevel.Error);
+ string error_msg = "Error: tag " + p.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update.";
+ bts.Log.Gen.error(error_msg);
return;
}
realBody = StreamDecompressor.decompressStream(bodyStream);
@@ -174,8 +156,9 @@ namespace BCG {
bodyStream.Read(trConstituent, 0, trConstituent.Length);
updates.Add(new PubSubUpdate(pub.tag, pub.type, new MemoryStream(trConstituent)));
}
- } else { }
- //BlueEventLog.logAssert(false, "Unknown message received: " + md.ToString() + " , only Pub and TransactionMetaData are legal");
+ } else {
+ bts.Log.assert_d(false, $"Unknown message received: {md} only Pub and TransactionMetaData are legal");
+ }
}
}
}
diff --git a/src/DataStreamSuite/PubSub/StaticSubscriberCore.cs b/src/DataStreamSuite/PubSub/StaticSubscriberCore.cs
index 1911ae8d3..fdcf2ff2e 100644
--- a/src/DataStreamSuite/PubSub/StaticSubscriberCore.cs
+++ b/src/DataStreamSuite/PubSub/StaticSubscriberCore.cs
@@ -80,7 +80,7 @@ namespace BCG {
int timeShare = timeout / myServerURLToSubCore.Count;
List retVal = new List();
foreach (ISingleServerSubscriberCore subCore in myServerURLToSubCore.Values)
- retVal.AddRange(subCore.getUpdates(timeShare));
+ retVal.AddRange(subCore.get_updates(timeShare));
_log_n_updates(retVal);
myTimer.stop();
//if (myTimer.lastIntervalTime > 10)
diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs
index 82586a5dd..14cea626b 100644
--- a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs
+++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs
@@ -1,26 +1,20 @@
-namespace BCG
-{
+namespace BCG {
using bts.api.orderman;
- enum TSGroupByOperation
- {
+ enum TSGroupByOperation {
OHLC = 0,
Statistics
}
[DataStreamUserRespresentation("timeseriesanalysis")]
- public class TimeSeriesGroupByStream : ViewStream
- {
- public TimeSeriesGroupByStream(ViewStream md_str, string timebucket_col, string output_col, string operation)
- {
+ public class TimeSeriesGroupByStream : ViewStream {
+ public TimeSeriesGroupByStream(ViewStream md_str, string timebucket_col, string output_col, string operation) {
ViewStream rs = new NonIncGroupByStream(md_str, new OHLCNonIncGroupByEvaluator(timebucket_col, output_col));
mySetRealSource(rs);
}
}
- public class OHLC
- {
- public OHLC(double open_args, double high_args, double low_args, double close_args)
- {
+ public class OHLC {
+ public OHLC(double open_args, double high_args, double low_args, double close_args) {
open = open_args;
high = high_args;
low = low_args;
@@ -28,47 +22,36 @@ namespace BCG
}
public readonly double open, high, low, close;
}
- public class OHLCNonIncGroupByEvaluator : INonIncGroupByEvaluator
- {
+ public class OHLCNonIncGroupByEvaluator : INonIncGroupByEvaluator {
private string _timestamp_col;
private string _output_col;
private IBlueTableSchema _out_schema;
- public OHLCNonIncGroupByEvaluator(string timestamp_col, string output_col)
- {
+ public OHLCNonIncGroupByEvaluator(string timestamp_col, string output_col) {
_timestamp_col = timestamp_col;
_output_col = output_col;
}
- string open_col
- {
- get
- {
+ string open_col {
+ get {
return $"{_output_col}_open"; ;
}
}
- string high_col
- {
- get
- {
+ string high_col {
+ get {
return $"{_output_col}_high"; ;
}
}
- string low_col
- {
- get
- {
+ string low_col {
+ get {
return $"{_output_col}_low"; ;
}
}
- string close_col
- {
- get
- {
+ string close_col {
+ get {
return $"{_output_col}_close"; ;
}
}
- private void _build_schema(IBlueTableSchema input_schema)
- {
+ private void _build_schema(IBlueTableSchema input_schema) {
List> new_cols = new List>();
ColumnInfo ci = input_schema[_output_col];
new_cols.Add(Tuple.Create(ci.colType, open_col));
@@ -79,32 +62,25 @@ namespace BCG
_out_schema = BlueDataUtility.add_columns(input_schema, new_cols.ToArray());
}
- public IBlueRowCollection evaluate(IDictionary> grouped_rows)
- {
- if (_out_schema == null)
- {
- if (grouped_rows.Count > 0)
- {
- foreach (var kv in grouped_rows)
- {
+ public IBlueRowCollection evaluate(IDictionary> grouped_rows) {
+ if (_out_schema == null) {
+ if (grouped_rows.Count > 0) {
+ foreach (var kv in grouped_rows) {
var rows = kv.Value;
- if (rows != null && rows.Count > 0)
- {
+ if (rows != null && rows.Count > 0) {
_build_schema(rows.ElementAt(0).schema);
}
}
}
}
BlueTable rv = new BlueTable(_out_schema);
- foreach (var kv in grouped_rows)
- {
+ foreach (var kv in grouped_rows) {
int id = kv.Key;
IEnumerable rows = kv.Value;
// sort rows on time to compute o,c correctly??
- if (rows != null && rows.Count() > 0)
- {
+ if (rows != null && rows.Count() > 0) {
IBlueRow reprow = rows.ElementAt(0);
- IEnumerable vals = BlueDataUtility.get_covalues(rows, _output_col);
+ IEnumerable vals = BlueDataUtility.get_colvalues(rows, _output_col);
OHLC ohlc = compute_ohlc(vals);
MutableBlueRow nr = new MutableBlueRow(_out_schema, reprow);
nr.setValue("id", kv.Key);
@@ -118,13 +94,11 @@ namespace BCG
return rv;
}
- public string[] get_group_by_cols()
- {
+ public string[] get_group_by_cols() {
return new string[] { _timestamp_col };
}
- static OHLC compute_ohlc(IEnumerable prices)
- {
+ static OHLC compute_ohlc(IEnumerable prices) {
double o = prices.ElementAt(0);
double c = prices.ElementAt(prices.Count() - 1);
double l = prices.Min();
@@ -133,28 +107,22 @@ namespace BCG
}
}
[DataStreamUserRespresentation("RSI")]
- public class RSIStream : ViewStream
- {
+ public class RSIStream : ViewStream {
// for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
- public RSIStream(ViewStream md_str, string timebucket_column, string output_col)
- {
+ public RSIStream(ViewStream md_str, string timebucket_column, string output_col) {
}
}
[DataStreamUserRespresentation("Bollinger")]
- public class BollingerBandStream : ViewStream
- {
+ public class BollingerBandStream : ViewStream {
// for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
- public BollingerBandStream(ViewStream md_str, string timebucket_column, string output_col)
- {
+ public BollingerBandStream(ViewStream md_str, string timebucket_column, string output_col) {
}
}
[DataStreamUserRespresentation("OHLC")]
- public class OHLCStream : ViewStream
- {
+ public class OHLCStream : ViewStream {
// for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for
- public OHLCStream(ViewStream md_str, string timebucket_column, string output_col)
- {
+ public OHLCStream(ViewStream md_str, string timebucket_column, string output_col) {
ViewStream rs = new TimeSeriesGroupByStream(md_str, timebucket_column, output_col, TSGroupByOperation.OHLC.ToString());
mySetRealSource(rs);
}