271 lines
12 KiB
C#
271 lines
12 KiB
C#
using BCG;
|
|
using BCG.Utilities.General;
|
|
|
|
using MongoDB.Bson;
|
|
using MongoDB.Driver;
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Diagnostics;
|
|
using System.Threading.Tasks;
|
|
using System.Threading;
|
|
|
|
namespace bts {
|
|
// [DataStreamUserRespresentation("mongodb_sql")]
|
|
// public class MongoDBSQLStream : ViewStream, IWorker {
|
|
// private ViewStreamDatabase _vsd;
|
|
// private readonly string _connection_string, _db_name, _collection_name;
|
|
// private MongoClient _client;
|
|
// // make this a sorted table
|
|
// private BlueTable _table;
|
|
|
|
|
|
// // Access the database
|
|
// private IMongoDatabase _database;
|
|
// private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
|
// // mongodb://192.168.1.80:27017
|
|
|
|
// public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
|
|
// : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
|
|
|
// }
|
|
// public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
|
// bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
|
// $"db_name: {db_name}");
|
|
// _connection_string = connection_string;
|
|
// _db_name = db_name;
|
|
// _collection_name = collection_name;
|
|
// _vsd = vsd;
|
|
// _client = new MongoClient(_connection_string);
|
|
// _database = _client.GetDatabase(_db_name);
|
|
// }
|
|
|
|
// protected override void myOnFirstSubscriber() {
|
|
// MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
|
|
// }
|
|
|
|
// protected override void myOnLastSubscriber() {
|
|
// MasterWorker.getInstance(false).removeWorker(this);
|
|
// }
|
|
|
|
// public void doWork() {
|
|
// _read_mogodb();
|
|
// }
|
|
|
|
// private void _read_mogodb() {
|
|
// var timer = Stopwatch.StartNew();
|
|
// // Access the collection
|
|
// IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
|
|
|
// // Fetch all documents from the collection
|
|
// var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
|
// _table = _table == null ? null : new BlueTable(_table.schema);
|
|
// // Iterate through the documents and print values
|
|
// foreach (var document in documents.ToEnumerable()) {
|
|
// bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
|
// if (_table == null) {
|
|
// _init_table(document);
|
|
// }
|
|
// _add_row(document);
|
|
// }
|
|
// timer.Stop();
|
|
// Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
|
|
// timer.Restart();
|
|
// var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
|
// timer.Stop();
|
|
// Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
|
|
// if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
|
// return;
|
|
// myFireEvent(dp);
|
|
// }
|
|
|
|
// private void _init_table(BsonDocument doc_row) {
|
|
// bts.Log.assert(_table == null);
|
|
// List<ColumnInfo> cis = new List<ColumnInfo>();
|
|
// cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
|
|
// foreach (var n in doc_row.Names) {
|
|
// cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
|
|
// }
|
|
// IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
|
|
// _table = new BlueTable(schema);
|
|
// }
|
|
|
|
// private void _add_row(BsonDocument doc) {
|
|
// var key_val = doc["_id"];
|
|
// var key = key_val.ToString();
|
|
// int id;
|
|
// if (!_id_map.TryGetValue(key, out id)) {
|
|
// id = _id_map.Count;
|
|
// _id_map.Add(key, id);
|
|
// }
|
|
// MutableBlueRow nr = new MutableBlueRow(_table.schema);
|
|
// nr.setValue("id", id);
|
|
// foreach (var n in doc.Names) {
|
|
// var val = doc[n].ToString();
|
|
// nr.setValue(n, val);
|
|
// }
|
|
// _table.addRow(nr);
|
|
// }
|
|
// }
|
|
|
|
[DataStreamUserRespresentation("mongodb")]
|
|
public class MongoDBStream : ViewStream, IWorker {
|
|
private ViewStreamDatabase _vsd;
|
|
private readonly string _connection_string, _db_name, _collection_name;
|
|
private readonly TimeSpan _poll_period;
|
|
private MongoClient _client;
|
|
private BlueTable _table;
|
|
private SyncQueue<DataPacket> _table_queue = new SyncQueue<DataPacket>();
|
|
private BlockingQueue _work_queue = new BlockingQueue();
|
|
|
|
// Access the database
|
|
private IMongoDatabase _database;
|
|
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
|
// mongodb://192.168.1.80:27017
|
|
|
|
public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period)
|
|
: this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) {
|
|
|
|
}
|
|
public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) {
|
|
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
|
$"db_name: {db_name}");
|
|
_connection_string = connection_string;
|
|
_db_name = db_name;
|
|
_collection_name = collection_name;
|
|
_vsd = vsd;
|
|
_client = new MongoClient(_connection_string);
|
|
_poll_period = poll_period;
|
|
_database = _client.GetDatabase(_db_name);
|
|
}
|
|
|
|
protected override void myOnFirstSubscriber() {
|
|
_vsd.addWorker(this);
|
|
//MasterWorker.getInstance(false).addWorker(this, _poll_period);
|
|
}
|
|
|
|
protected override void myOnLastSubscriber() {
|
|
_vsd.removeWorker(this);
|
|
//MasterWorker.getInstance(false).removeWorker(this);
|
|
}
|
|
|
|
private DateTime _last_poll_time = DateTime.MinValue;
|
|
public void doWork() {
|
|
var now = DateTime.Now;
|
|
if (now - _last_poll_time > _poll_period) {
|
|
ThreadPool.QueueUserWorkItem(_ => _read_mogodb());
|
|
_last_poll_time = DateTime.Now;
|
|
}
|
|
var dp = _table_queue.dequeue(0);
|
|
if (dp != null)
|
|
myFireEvent(dp);
|
|
}
|
|
|
|
private void _read_mogodb() {
|
|
var timer = Stopwatch.StartNew();
|
|
// Access the collection
|
|
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
|
|
|
// Fetch all documents from the collection
|
|
var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
|
_table = _table == null ? null : new BlueTable(_table.schema);
|
|
// Iterate through the documents and print values
|
|
foreach (var document in documents.ToEnumerable()) {
|
|
//bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
|
if (_table == null) {
|
|
_init_table(document);
|
|
}
|
|
_add_doc(document);
|
|
}
|
|
timer.Stop();
|
|
Log.Gen.info($"MongoDBStream time to read {_collection_name}: {timer.ElapsedMilliseconds}ms");
|
|
timer.Restart();
|
|
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
|
timer.Stop();
|
|
Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ;
|
|
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
|
return;
|
|
_table_queue.enqueue(dp);
|
|
//myFireEvent(dp);
|
|
}
|
|
|
|
private void _init_table(BsonDocument doc_row) {
|
|
bts.Log.assert(_table == null);
|
|
var schema = BlueDataUtility.form_schema($"{_database}_{_table}",
|
|
Tuple.Create("id", TypeCode.Int32),
|
|
Tuple.Create("doc_id", TypeCode.String),
|
|
Tuple.Create("key", TypeCode.String),
|
|
Tuple.Create("value", TypeCode.String));
|
|
_table = new BlueTable(schema);
|
|
}
|
|
|
|
private void _add_doc(BsonDocument doc) {
|
|
var doc_id_bson = doc["_id"];
|
|
var doc_id = doc_id_bson.ToString();
|
|
Func<string, string, int> get_id = (string doc_id, string key) => {
|
|
string hash_key = $"{doc_id}_{key}";
|
|
int id = 0;
|
|
if (!_id_map.TryGetValue(hash_key, out id)) {
|
|
id = _id_map.Count;
|
|
_id_map.Add(hash_key, id);
|
|
}
|
|
return id;
|
|
};
|
|
foreach (var n in doc.Names) {
|
|
string key = n;
|
|
var id = get_id(doc_id, key);
|
|
MutableBlueRow nr = new MutableBlueRow(_table.schema);
|
|
var val = doc[n].ToString();
|
|
nr.setValue("id", id);
|
|
nr.setValue("doc_id", doc_id);
|
|
nr.setValue("key", key);
|
|
nr.setValue("value", val);
|
|
_table.addRow(nr);
|
|
}
|
|
}
|
|
}
|
|
|
|
// [DataStreamUserRespresentation("mongodb_flat")]
|
|
// public class MongoDBFlatStream : ViewStream {
|
|
// public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
|
|
// :
|
|
// this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
|
|
|
|
// public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
|
|
// ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
|
// rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
|
|
// rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
|
|
// //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
|
|
// mySetRealSource(rs);
|
|
// }
|
|
// }
|
|
|
|
|
|
|
|
// [DataStreamUserRespresentation("newsreplay")]
|
|
// public class NewsReplayStream : ViewStream {
|
|
// public NewsReplayStream(string connection_string, string db_name, string collection_name,
|
|
// int buffer_sz, string fire_period) :
|
|
// this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
|
|
|
|
// public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
|
|
// int buffer_sz, TimeSpan fire_period) {
|
|
// ViewStream r;
|
|
|
|
// r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
|
// r = new KeyValueToRelationalStream(r,
|
|
// "doc_id",
|
|
// "key",
|
|
// "value");
|
|
// r = new DateStandardizationStream(r,
|
|
// "created_at",
|
|
// "ISO8601");
|
|
// r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
|
|
|
|
// mySetRealSource(r);
|
|
// }
|
|
// }
|
|
|
|
|
|
}
|