Files
spark/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs
T
2024-06-06 09:20:05 -05:00

254 lines
11 KiB
C#

using BCG;
using BCG.Utilities.General;
using MongoDB.Bson;
using MongoDB.Driver;
using System;
using System.Collections.Generic;
using System.Diagnostics;
namespace bts {
[DataStreamUserRespresentation("mongodb_sql")]
public class MongoDBSQLStream : ViewStream, IWorker {
private ViewStreamDatabase _vsd;
private readonly string _connection_string, _db_name, _collection_name;
private MongoClient _client;
// make this a sorted table
private BlueTable _table;
// Access the database
private IMongoDatabase _database;
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
// mongodb://192.168.1.80:27017
public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
}
public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
$"db_name: {db_name}");
_connection_string = connection_string;
_db_name = db_name;
_collection_name = collection_name;
_vsd = vsd;
_client = new MongoClient(_connection_string);
_database = _client.GetDatabase(_db_name);
}
protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
}
protected override void myOnLastSubscriber() {
MasterWorker.getInstance(false).removeWorker(this);
}
public void doWork() {
_read_mogodb();
}
private void _read_mogodb() {
var timer = Stopwatch.StartNew();
// Access the collection
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
// Fetch all documents from the collection
var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
_table = _table == null ? null : new BlueTable(_table.schema);
// Iterate through the documents and print values
foreach (var document in documents.ToEnumerable()) {
bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
if (_table == null) {
_init_table(document);
}
_add_row(document);
}
timer.Stop();
Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
timer.Restart();
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
timer.Stop();
Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
return;
myFireEvent(dp);
}
private void _init_table(BsonDocument doc_row) {
bts.Log.assert(_table == null);
List<ColumnInfo> cis = new List<ColumnInfo>();
cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
foreach (var n in doc_row.Names) {
cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
}
IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
_table = new BlueTable(schema);
}
private void _add_row(BsonDocument doc) {
var key_val = doc["_id"];
var key = key_val.ToString();
int id;
if (!_id_map.TryGetValue(key, out id)) {
id = _id_map.Count;
_id_map.Add(key, id);
}
MutableBlueRow nr = new MutableBlueRow(_table.schema);
nr.setValue("id", id);
foreach (var n in doc.Names) {
var val = doc[n].ToString();
nr.setValue(n, val);
}
_table.addRow(nr);
}
}
[DataStreamUserRespresentation("mongodb")]
public class MongoDBStream : ViewStream, IWorker {
private ViewStreamDatabase _vsd;
private readonly string _connection_string, _db_name, _collection_name;
private MongoClient _client;
private BlueTable _table;
// Access the database
private IMongoDatabase _database;
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
// mongodb://192.168.1.80:27017
public MongoDBStream(string connection_string, string db_name, string collection_name)
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
}
public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
$"db_name: {db_name}");
_connection_string = connection_string;
_db_name = db_name;
_collection_name = collection_name;
_vsd = vsd;
_client = new MongoClient(_connection_string);
_database = _client.GetDatabase(_db_name);
}
protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
}
protected override void myOnLastSubscriber() {
MasterWorker.getInstance(false).removeWorker(this);
}
public void doWork() {
_read_mogodb();
}
private void _read_mogodb() {
var timer = Stopwatch.StartNew();
// Access the collection
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
// Fetch all documents from the collection
var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
_table = _table == null ? null : new BlueTable(_table.schema);
// Iterate through the documents and print values
foreach (var document in documents.ToEnumerable()) {
//bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
if (_table == null) {
_init_table(document);
}
_add_doc(document);
}
timer.Stop();
Log.Gen.info($"MongoDBStream time to read {_collection_name}: {timer.ElapsedMilliseconds}ms");
timer.Restart();
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
timer.Stop();
Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms");;
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
return;
myFireEvent(dp);
}
private void _init_table(BsonDocument doc_row) {
bts.Log.assert(_table == null);
var schema = BlueDataUtility.form_schema($"{_database}_{_table}",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("doc_id", TypeCode.String),
Tuple.Create("key", TypeCode.String),
Tuple.Create("value", TypeCode.String));
_table = new BlueTable(schema);
}
private void _add_doc(BsonDocument doc) {
var doc_id_bson = doc["_id"];
var doc_id = doc_id_bson.ToString();
Func<string, string, int> get_id = (string doc_id, string key) => {
string hash_key = $"{doc_id}_{key}";
int id = 0;
if (!_id_map.TryGetValue(hash_key, out id)) {
id = _id_map.Count;
_id_map.Add(hash_key, id);
}
return id;
};
foreach (var n in doc.Names) {
string key = n;
var id = get_id(doc_id, key);
MutableBlueRow nr = new MutableBlueRow(_table.schema);
var val = doc[n].ToString();
nr.setValue("id", id);
nr.setValue("doc_id", doc_id);
nr.setValue("key", key);
nr.setValue("value", val);
_table.addRow(nr);
}
}
}
[DataStreamUserRespresentation("mongodb_flat")]
public class MongoDBFlatStream : ViewStream {
public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
:
this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
//rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
mySetRealSource(rs);
}
}
[DataStreamUserRespresentation("newsreplay")]
public class NewsReplayStream : ViewStream {
public NewsReplayStream(string connection_string, string db_name, string collection_name,
int buffer_sz, string fire_period) :
this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
int buffer_sz, TimeSpan fire_period) {
ViewStream r;
r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
r = new KeyValueToRelationalStream(r,
"doc_id",
"key",
"value");
r = new DateStandardizationStream(r,
"created_at",
"ISO8601");
r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
mySetRealSource(r);
}
}
}