diff --git a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs index 52015e568..6da6ed12a 100644 --- a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs @@ -1,212 +1,210 @@ -using System; -using System.Collections.Generic; -using MongoDB.Driver; -using MongoDB.Bson; -using BCG; +namespace BCG; + using BCG.Utilities.General; -using ZstdSharp.Unsafe; -using System.Runtime.CompilerServices; + using mbts.spark; +using MongoDB.Bson; +using MongoDB.Driver; -namespace BCG { - [DataStreamUserRespresentation("mongodb_sql")] - public class MongoDBSQLStream : ViewStream, IWorker { - private ViewStreamDatabase _vsd; - private readonly string _connection_string, _db_name, _collection_name; - private MongoClient _client; - private BlueTable _table; +using System; +using System.Collections.Generic; - // Access the database - private IMongoDatabase _database; - private IDictionary _id_map = new Dictionary(); - // mongodb://192.168.1.80:27017 +[DataStreamUserRespresentation("mongodb_sql")] +public class MongoDBSQLStream : ViewStream, IWorker { + private ViewStreamDatabase _vsd; + private readonly string _connection_string, _db_name, _collection_name; + private MongoClient _client; + private BlueTable _table; - public MongoDBSQLStream(string connection_string, string db_name, string collection_name) - : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) { + // Access the database + private IMongoDatabase _database; + private IDictionary _id_map = new Dictionary(); + // mongodb://192.168.1.80:27017 - } - public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) { - bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " + - $"db_name: {db_name}"); - _connection_string = connection_string; - _db_name = db_name; - _collection_name = collection_name; - _vsd = vsd; - _client = new MongoClient(_connection_string); - _database = _client.GetDatabase(_db_name); - } + public MongoDBSQLStream(string connection_string, string db_name, string collection_name) + : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) { - protected override void myOnFirstSubscriber() { - MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); - } + } + public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) { + bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " + + $"db_name: {db_name}"); + _connection_string = connection_string; + _db_name = db_name; + _collection_name = collection_name; + _vsd = vsd; + _client = new MongoClient(_connection_string); + _database = _client.GetDatabase(_db_name); + } - protected override void myOnLastSubscriber() { - MasterWorker.getInstance(false).removeWorker(this); - } + protected override void myOnFirstSubscriber() { + MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); + } - public void doWork() { - _read_mogodb(); - } + protected override void myOnLastSubscriber() { + MasterWorker.getInstance(false).removeWorker(this); + } - private void _read_mogodb() { - HighPerformanceTimer timer = new HighPerformanceTimer(); - timer.start(); - // Access the collection - IMongoCollection collection = _database.GetCollection(name: _collection_name); + public void doWork() { + _read_mogodb(); + } - // Fetch all documents from the collection - var documents = collection.Find(Builders.Filter.Empty).ToCursor(); - _table = _table == null ? null : new BlueTable(_table.schema); - // Iterate through the documents and print values - foreach (var document in documents.ToEnumerable()) { - bts.Log.Gen.info($"document: {document} json: {document.ToJson()}"); - if (_table == null) { - _init_table(document); - } - _add_row(document); - } - timer.stop(); - bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs"); - timer.start(); - var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true); - timer.stop(); - bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs"); - if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0) - return; - myFireEvent(dp); - } + private void _read_mogodb() { + HighPerformanceTimer timer = new HighPerformanceTimer(); + timer.start(); + // Access the collection + IMongoCollection collection = _database.GetCollection(name: _collection_name); - private void _init_table(BsonDocument doc_row) { - bts.Log.assert(_table == null); - List cis = new List(); - cis.Add((new ColumnInfo("id", 0, typeof(Int32)))); - foreach (var n in doc_row.Names) { - cis.Add(new ColumnInfo(n, cis.Count, typeof(string))); - } - IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name); - _table = new BlueTable(schema); - } + // Fetch all documents from the collection + var documents = collection.Find(Builders.Filter.Empty).ToCursor(); + _table = _table == null ? null : new BlueTable(_table.schema); + // Iterate through the documents and print values + foreach (var document in documents.ToEnumerable()) { + bts.Log.Gen.info($"document: {document} json: {document.ToJson()}"); + if (_table == null) { + _init_table(document); + } + _add_row(document); + } + timer.stop(); + bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs"); + timer.start(); + var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true); + timer.stop(); + bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs"); + if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0) + return; + myFireEvent(dp); + } - private void _add_row(BsonDocument doc) { - var key_val = doc["_id"]; - var key = key_val.ToString(); - int id; - if (!_id_map.TryGetValue(key, out id)) { - id = _id_map.Count; - _id_map.Add(key, id); - } - MutableBlueRow nr = new MutableBlueRow(_table.schema); - nr.setValue("id", id); - foreach (var n in doc.Names) { - var val = doc[n].ToString(); - nr.setValue(n, val); - } - _table.addRow(nr); - } - } + private void _init_table(BsonDocument doc_row) { + bts.Log.assert(_table == null); + List cis = new List(); + cis.Add((new ColumnInfo("id", 0, typeof(Int32)))); + foreach (var n in doc_row.Names) { + cis.Add(new ColumnInfo(n, cis.Count, typeof(string))); + } + IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name); + _table = new BlueTable(schema); + } - [DataStreamUserRespresentation("mongodb")] - public class MongoDBStream : ViewStream, IWorker { - private ViewStreamDatabase _vsd; - private readonly string _connection_string, _db_name, _collection_name; - private MongoClient _client; - private BlueTable _table; + private void _add_row(BsonDocument doc) { + var key_val = doc["_id"]; + var key = key_val.ToString(); + int id; + if (!_id_map.TryGetValue(key, out id)) { + id = _id_map.Count; + _id_map.Add(key, id); + } + MutableBlueRow nr = new MutableBlueRow(_table.schema); + nr.setValue("id", id); + foreach (var n in doc.Names) { + var val = doc[n].ToString(); + nr.setValue(n, val); + } + _table.addRow(nr); + } +} - // Access the database - private IMongoDatabase _database; - private IDictionary _id_map = new Dictionary(); - // mongodb://192.168.1.80:27017 +[DataStreamUserRespresentation("mongodb")] +public class MongoDBStream : ViewStream, IWorker { + private ViewStreamDatabase _vsd; + private readonly string _connection_string, _db_name, _collection_name; + private MongoClient _client; + private BlueTable _table; - public MongoDBStream(string connection_string, string db_name, string collection_name) - : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) { + // Access the database + private IMongoDatabase _database; + private IDictionary _id_map = new Dictionary(); + // mongodb://192.168.1.80:27017 - } - public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) { - bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " + - $"db_name: {db_name}"); - _connection_string = connection_string; - _db_name = db_name; - _collection_name = collection_name; - _vsd = vsd; - _client = new MongoClient(_connection_string); - _database = _client.GetDatabase(_db_name); - } + public MongoDBStream(string connection_string, string db_name, string collection_name) + : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) { - protected override void myOnFirstSubscriber() { - MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); - } + } + public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) { + bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " + + $"db_name: {db_name}"); + _connection_string = connection_string; + _db_name = db_name; + _collection_name = collection_name; + _vsd = vsd; + _client = new MongoClient(_connection_string); + _database = _client.GetDatabase(_db_name); + } - protected override void myOnLastSubscriber() { - MasterWorker.getInstance(false).removeWorker(this); - } + protected override void myOnFirstSubscriber() { + MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); + } - public void doWork() { - _read_mogodb(); - } + protected override void myOnLastSubscriber() { + MasterWorker.getInstance(false).removeWorker(this); + } - private void _read_mogodb() { - HighPerformanceTimer timer = new HighPerformanceTimer(); - timer.start(); - // Access the collection - IMongoCollection collection = _database.GetCollection(name: _collection_name); + public void doWork() { + _read_mogodb(); + } - // Fetch all documents from the collection - var documents = collection.Find(Builders.Filter.Empty).ToCursor(); - _table = _table == null ? null : new BlueTable(_table.schema); - // Iterate through the documents and print values - foreach (var document in documents.ToEnumerable()) { - bts.Log.Gen.info($"document: {document} json: {document.ToJson()}"); - if (_table == null) { - _init_table(document); - } - _add_doc(document); - } - timer.stop(); - bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs"); - timer.start(); - var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true); - timer.stop(); - bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs"); - if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0) - return; - myFireEvent(dp); - } + private void _read_mogodb() { + HighPerformanceTimer timer = new HighPerformanceTimer(); + timer.start(); + // Access the collection + IMongoCollection collection = _database.GetCollection(name: _collection_name); - private void _init_table(BsonDocument doc_row) { - bts.Log.assert(_table == null); - var schema = BlueDataUtility.form_schema($"{_database}_{_table}", - Tuple.Create("id", TypeCode.Int32), - Tuple.Create("doc_id", TypeCode.String), - Tuple.Create("key", TypeCode.String), - Tuple.Create("value", TypeCode.String)); - _table = new BlueTable(schema); - } + // Fetch all documents from the collection + var documents = collection.Find(Builders.Filter.Empty).ToCursor(); + _table = _table == null ? null : new BlueTable(_table.schema); + // Iterate through the documents and print values + foreach (var document in documents.ToEnumerable()) { + bts.Log.Gen.info($"document: {document} json: {document.ToJson()}"); + if (_table == null) { + _init_table(document); + } + _add_doc(document); + } + timer.stop(); + bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs"); + timer.start(); + var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true); + timer.stop(); + bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs"); + if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0) + return; + myFireEvent(dp); + } - private void _add_doc(BsonDocument doc) { - var doc_id_bson = doc["_id"]; - var doc_id = doc_id_bson.ToString(); - Func get_id = (string doc_id, string key) => { - string hash_key = $"{doc_id}_{key}"; - int id = 0; - if (!_id_map.TryGetValue(hash_key, out id)) { - id = _id_map.Count; - _id_map.Add(hash_key, id); - } - return id; - }; - foreach (var n in doc.Names) { - string key = n; - var id = get_id(doc_id, key); - MutableBlueRow nr = new MutableBlueRow(_table.schema); - var val = doc[n].ToString(); - nr.setValue("id", id); - nr.setValue("doc_id", doc_id); - nr.setValue("key", key); - nr.setValue("value", val); - _table.addRow(nr); - } - } - } + private void _init_table(BsonDocument doc_row) { + bts.Log.assert(_table == null); + var schema = BlueDataUtility.form_schema($"{_database}_{_table}", + Tuple.Create("id", TypeCode.Int32), + Tuple.Create("doc_id", TypeCode.String), + Tuple.Create("key", TypeCode.String), + Tuple.Create("value", TypeCode.String)); + _table = new BlueTable(schema); + } -} \ No newline at end of file + private void _add_doc(BsonDocument doc) { + var doc_id_bson = doc["_id"]; + var doc_id = doc_id_bson.ToString(); + Func get_id = (string doc_id, string key) => { + string hash_key = $"{doc_id}_{key}"; + int id = 0; + if (!_id_map.TryGetValue(hash_key, out id)) { + id = _id_map.Count; + _id_map.Add(hash_key, id); + } + return id; + }; + foreach (var n in doc.Names) { + string key = n; + var id = get_id(doc_id, key); + MutableBlueRow nr = new MutableBlueRow(_table.schema); + var val = doc[n].ToString(); + nr.setValue("id", id); + nr.setValue("doc_id", doc_id); + nr.setValue("key", key); + nr.setValue("value", val); + _table.addRow(nr); + } + } +}