using BCG; using BCG.Utilities.General; using MongoDB.Bson; using MongoDB.Driver; using System; using System.Collections.Generic; using System.Diagnostics; using System.Threading.Tasks; using System.Threading; namespace bts { // [DataStreamUserRespresentation("mongodb_sql")] // public class MongoDBSQLStream : ViewStream, IWorker { // private ViewStreamDatabase _vsd; // private readonly string _connection_string, _db_name, _collection_name; // private MongoClient _client; // // make this a sorted table // private BlueTable _table; // // Access the database // private IMongoDatabase _database; // private IDictionary _id_map = new Dictionary(); // // mongodb://192.168.1.80:27017 // public MongoDBSQLStream(string connection_string, string db_name, string collection_name) // : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) { // } // public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) { // bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " + // $"db_name: {db_name}"); // _connection_string = connection_string; // _db_name = db_name; // _collection_name = collection_name; // _vsd = vsd; // _client = new MongoClient(_connection_string); // _database = _client.GetDatabase(_db_name); // } // protected override void myOnFirstSubscriber() { // MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10)); // } // protected override void myOnLastSubscriber() { // MasterWorker.getInstance(false).removeWorker(this); // } // public void doWork() { // _read_mogodb(); // } // private void _read_mogodb() { // var timer = Stopwatch.StartNew(); // // Access the collection // IMongoCollection collection = _database.GetCollection(name: _collection_name); // // Fetch all documents from the collection // var documents = collection.Find(Builders.Filter.Empty).ToCursor(); // _table = _table == null ? null : new BlueTable(_table.schema); // // Iterate through the documents and print values // foreach (var document in documents.ToEnumerable()) { // bts.Log.Gen.info($"document: {document} json: {document.ToJson()}"); // if (_table == null) { // _init_table(document); // } // _add_row(document); // } // timer.Stop(); // Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms"); // timer.Restart(); // var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true); // timer.Stop(); // Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms"); // if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0) // return; // myFireEvent(dp); // } // private void _init_table(BsonDocument doc_row) { // bts.Log.assert(_table == null); // List cis = new List(); // cis.Add((new ColumnInfo("id", 0, typeof(Int32)))); // foreach (var n in doc_row.Names) { // cis.Add(new ColumnInfo(n, cis.Count, typeof(string))); // } // IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name); // _table = new BlueTable(schema); // } // private void _add_row(BsonDocument doc) { // var key_val = doc["_id"]; // var key = key_val.ToString(); // int id; // if (!_id_map.TryGetValue(key, out id)) { // id = _id_map.Count; // _id_map.Add(key, id); // } // MutableBlueRow nr = new MutableBlueRow(_table.schema); // nr.setValue("id", id); // foreach (var n in doc.Names) { // var val = doc[n].ToString(); // nr.setValue(n, val); // } // _table.addRow(nr); // } // } [DataStreamUserRespresentation("mongodb")] public class MongoDBStream : ViewStream, IWorker { private ViewStreamDatabase _vsd; private readonly string _connection_string, _db_name, _collection_name; private readonly TimeSpan _poll_period; private MongoClient _client; private BlueTable _table; private SyncQueue _table_queue = new SyncQueue(); private BlockingQueue _work_queue = new BlockingQueue(); // Access the database private IMongoDatabase _database; private IDictionary _id_map = new Dictionary(); // mongodb://192.168.1.80:27017 public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period) : this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) { } public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) { bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " + $"db_name: {db_name}"); _connection_string = connection_string; _db_name = db_name; _collection_name = collection_name; _vsd = vsd; _client = new MongoClient(_connection_string); _poll_period = poll_period; _database = _client.GetDatabase(_db_name); } protected override void myOnFirstSubscriber() { _vsd.addWorker(this); //MasterWorker.getInstance(false).addWorker(this, _poll_period); } protected override void myOnLastSubscriber() { _vsd.removeWorker(this); //MasterWorker.getInstance(false).removeWorker(this); } private DateTime _last_poll_time = DateTime.MinValue; public void doWork() { var now = DateTime.Now; if (now - _last_poll_time > _poll_period) { ThreadPool.QueueUserWorkItem(_ => _read_mogodb()); _last_poll_time = DateTime.Now; } var dp = _table_queue.dequeue(0); if (dp != null) myFireEvent(dp); } private void _read_mogodb() { var timer = Stopwatch.StartNew(); // Access the collection IMongoCollection collection = _database.GetCollection(name: _collection_name); // Fetch all documents from the collection var documents = collection.Find(Builders.Filter.Empty).ToCursor(); _table = _table == null ? null : new BlueTable(_table.schema); // Iterate through the documents and print values foreach (var document in documents.ToEnumerable()) { //bts.Log.Gen.info($"document: {document} json: {document.ToJson()}"); if (_table == null) { _init_table(document); } _add_doc(document); } timer.Stop(); Log.Gen.info($"MongoDBStream time to read {_collection_name}: {timer.ElapsedMilliseconds}ms"); timer.Restart(); var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true); timer.Stop(); Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ; if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0) return; _table_queue.enqueue(dp); //myFireEvent(dp); } private void _init_table(BsonDocument doc_row) { bts.Log.assert(_table == null); var schema = BlueDataUtility.form_schema($"{_database}_{_table}", Tuple.Create("id", TypeCode.Int32), Tuple.Create("doc_id", TypeCode.String), Tuple.Create("key", TypeCode.String), Tuple.Create("value", TypeCode.String)); _table = new BlueTable(schema); } private void _add_doc(BsonDocument doc) { var doc_id_bson = doc["_id"]; var doc_id = doc_id_bson.ToString(); Func get_id = (string doc_id, string key) => { string hash_key = $"{doc_id}_{key}"; int id = 0; if (!_id_map.TryGetValue(hash_key, out id)) { id = _id_map.Count; _id_map.Add(hash_key, id); } return id; }; foreach (var n in doc.Names) { string key = n; var id = get_id(doc_id, key); MutableBlueRow nr = new MutableBlueRow(_table.schema); var val = doc[n].ToString(); nr.setValue("id", id); nr.setValue("doc_id", doc_id); nr.setValue("key", key); nr.setValue("value", val); _table.addRow(nr); } } } // [DataStreamUserRespresentation("mongodb_flat")] // public class MongoDBFlatStream : ViewStream { // public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration) // : // this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { } // public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) { // ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd); // rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val"); // rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601); // //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration); // mySetRealSource(rs); // } // } // [DataStreamUserRespresentation("newsreplay")] // public class NewsReplayStream : ViewStream { // public NewsReplayStream(string connection_string, string db_name, string collection_name, // int buffer_sz, string fire_period) : // this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { } // public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, // int buffer_sz, TimeSpan fire_period) { // ViewStream r; // r = new MongoDBStream(connection_string, db_name, collection_name, vsd); // r = new KeyValueToRelationalStream(r, // "doc_id", // "key", // "value"); // r = new DateStandardizationStream(r, // "created_at", // "ISO8601"); // r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period); // mySetRealSource(r); // } // } }