Normalize indentation in MongoDBStream
This commit is contained in:
1 parent
d6a744e5ab
commit
6a244fee35
1 file changed
+184
-186
@@ -1,212 +1,210 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using MongoDB.Driver;
|
||||
using MongoDB.Bson;
|
||||
using BCG;
|
||||
namespace BCG;
|
||||
|
||||
using BCG.Utilities.General;
|
||||
using ZstdSharp.Unsafe;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
using mbts.spark;
|
||||
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace BCG {
|
||||
[DataStreamUserRespresentation("mongodb_sql")]
|
||||
public class MongoDBSQLStream : ViewStream, IWorker {
|
||||
private ViewStreamDatabase _vsd;
|
||||
private readonly string _connection_string, _db_name, _collection_name;
|
||||
private MongoClient _client;
|
||||
private BlueTable _table;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
// Access the database
|
||||
private IMongoDatabase _database;
|
||||
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
||||
// mongodb://192.168.1.80:27017
|
||||
[DataStreamUserRespresentation("mongodb_sql")]
|
||||
public class MongoDBSQLStream : ViewStream, IWorker {
|
||||
private ViewStreamDatabase _vsd;
|
||||
private readonly string _connection_string, _db_name, _collection_name;
|
||||
private MongoClient _client;
|
||||
private BlueTable _table;
|
||||
|
||||
public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
|
||||
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
||||
// Access the database
|
||||
private IMongoDatabase _database;
|
||||
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
||||
// mongodb://192.168.1.80:27017
|
||||
|
||||
}
|
||||
public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
||||
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
||||
$"db_name: {db_name}");
|
||||
_connection_string = connection_string;
|
||||
_db_name = db_name;
|
||||
_collection_name = collection_name;
|
||||
_vsd = vsd;
|
||||
_client = new MongoClient(_connection_string);
|
||||
_database = _client.GetDatabase(_db_name);
|
||||
}
|
||||
public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
|
||||
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
|
||||
}
|
||||
}
|
||||
public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
||||
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
||||
$"db_name: {db_name}");
|
||||
_connection_string = connection_string;
|
||||
_db_name = db_name;
|
||||
_collection_name = collection_name;
|
||||
_vsd = vsd;
|
||||
_client = new MongoClient(_connection_string);
|
||||
_database = _client.GetDatabase(_db_name);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
MasterWorker.getInstance(false).removeWorker(this);
|
||||
}
|
||||
protected override void myOnFirstSubscriber() {
|
||||
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
|
||||
}
|
||||
|
||||
public void doWork() {
|
||||
_read_mogodb();
|
||||
}
|
||||
protected override void myOnLastSubscriber() {
|
||||
MasterWorker.getInstance(false).removeWorker(this);
|
||||
}
|
||||
|
||||
private void _read_mogodb() {
|
||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||
timer.start();
|
||||
// Access the collection
|
||||
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
||||
public void doWork() {
|
||||
_read_mogodb();
|
||||
}
|
||||
|
||||
// Fetch all documents from the collection
|
||||
var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
||||
_table = _table == null ? null : new BlueTable(_table.schema);
|
||||
// Iterate through the documents and print values
|
||||
foreach (var document in documents.ToEnumerable()) {
|
||||
bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
||||
if (_table == null) {
|
||||
_init_table(document);
|
||||
}
|
||||
_add_row(document);
|
||||
}
|
||||
timer.stop();
|
||||
bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
|
||||
timer.start();
|
||||
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||
timer.stop();
|
||||
bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
|
||||
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||
return;
|
||||
myFireEvent(dp);
|
||||
}
|
||||
private void _read_mogodb() {
|
||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||
timer.start();
|
||||
// Access the collection
|
||||
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
||||
|
||||
private void _init_table(BsonDocument doc_row) {
|
||||
bts.Log.assert(_table == null);
|
||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||
cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
|
||||
foreach (var n in doc_row.Names) {
|
||||
cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
|
||||
}
|
||||
IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
|
||||
_table = new BlueTable(schema);
|
||||
}
|
||||
// Fetch all documents from the collection
|
||||
var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
||||
_table = _table == null ? null : new BlueTable(_table.schema);
|
||||
// Iterate through the documents and print values
|
||||
foreach (var document in documents.ToEnumerable()) {
|
||||
bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
||||
if (_table == null) {
|
||||
_init_table(document);
|
||||
}
|
||||
_add_row(document);
|
||||
}
|
||||
timer.stop();
|
||||
bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
|
||||
timer.start();
|
||||
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||
timer.stop();
|
||||
bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
|
||||
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||
return;
|
||||
myFireEvent(dp);
|
||||
}
|
||||
|
||||
private void _add_row(BsonDocument doc) {
|
||||
var key_val = doc["_id"];
|
||||
var key = key_val.ToString();
|
||||
int id;
|
||||
if (!_id_map.TryGetValue(key, out id)) {
|
||||
id = _id_map.Count;
|
||||
_id_map.Add(key, id);
|
||||
}
|
||||
MutableBlueRow nr = new MutableBlueRow(_table.schema);
|
||||
nr.setValue("id", id);
|
||||
foreach (var n in doc.Names) {
|
||||
var val = doc[n].ToString();
|
||||
nr.setValue(n, val);
|
||||
}
|
||||
_table.addRow(nr);
|
||||
}
|
||||
}
|
||||
private void _init_table(BsonDocument doc_row) {
|
||||
bts.Log.assert(_table == null);
|
||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||
cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
|
||||
foreach (var n in doc_row.Names) {
|
||||
cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
|
||||
}
|
||||
IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
|
||||
_table = new BlueTable(schema);
|
||||
}
|
||||
|
||||
[DataStreamUserRespresentation("mongodb")]
|
||||
public class MongoDBStream : ViewStream, IWorker {
|
||||
private ViewStreamDatabase _vsd;
|
||||
private readonly string _connection_string, _db_name, _collection_name;
|
||||
private MongoClient _client;
|
||||
private BlueTable _table;
|
||||
private void _add_row(BsonDocument doc) {
|
||||
var key_val = doc["_id"];
|
||||
var key = key_val.ToString();
|
||||
int id;
|
||||
if (!_id_map.TryGetValue(key, out id)) {
|
||||
id = _id_map.Count;
|
||||
_id_map.Add(key, id);
|
||||
}
|
||||
MutableBlueRow nr = new MutableBlueRow(_table.schema);
|
||||
nr.setValue("id", id);
|
||||
foreach (var n in doc.Names) {
|
||||
var val = doc[n].ToString();
|
||||
nr.setValue(n, val);
|
||||
}
|
||||
_table.addRow(nr);
|
||||
}
|
||||
}
|
||||
|
||||
// Access the database
|
||||
private IMongoDatabase _database;
|
||||
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
||||
// mongodb://192.168.1.80:27017
|
||||
[DataStreamUserRespresentation("mongodb")]
|
||||
public class MongoDBStream : ViewStream, IWorker {
|
||||
private ViewStreamDatabase _vsd;
|
||||
private readonly string _connection_string, _db_name, _collection_name;
|
||||
private MongoClient _client;
|
||||
private BlueTable _table;
|
||||
|
||||
public MongoDBStream(string connection_string, string db_name, string collection_name)
|
||||
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
||||
// Access the database
|
||||
private IMongoDatabase _database;
|
||||
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
||||
// mongodb://192.168.1.80:27017
|
||||
|
||||
}
|
||||
public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
||||
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
||||
$"db_name: {db_name}");
|
||||
_connection_string = connection_string;
|
||||
_db_name = db_name;
|
||||
_collection_name = collection_name;
|
||||
_vsd = vsd;
|
||||
_client = new MongoClient(_connection_string);
|
||||
_database = _client.GetDatabase(_db_name);
|
||||
}
|
||||
public MongoDBStream(string connection_string, string db_name, string collection_name)
|
||||
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
|
||||
}
|
||||
}
|
||||
public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
||||
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
||||
$"db_name: {db_name}");
|
||||
_connection_string = connection_string;
|
||||
_db_name = db_name;
|
||||
_collection_name = collection_name;
|
||||
_vsd = vsd;
|
||||
_client = new MongoClient(_connection_string);
|
||||
_database = _client.GetDatabase(_db_name);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
MasterWorker.getInstance(false).removeWorker(this);
|
||||
}
|
||||
protected override void myOnFirstSubscriber() {
|
||||
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
|
||||
}
|
||||
|
||||
public void doWork() {
|
||||
_read_mogodb();
|
||||
}
|
||||
protected override void myOnLastSubscriber() {
|
||||
MasterWorker.getInstance(false).removeWorker(this);
|
||||
}
|
||||
|
||||
private void _read_mogodb() {
|
||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||
timer.start();
|
||||
// Access the collection
|
||||
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
||||
public void doWork() {
|
||||
_read_mogodb();
|
||||
}
|
||||
|
||||
// Fetch all documents from the collection
|
||||
var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
||||
_table = _table == null ? null : new BlueTable(_table.schema);
|
||||
// Iterate through the documents and print values
|
||||
foreach (var document in documents.ToEnumerable()) {
|
||||
bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
||||
if (_table == null) {
|
||||
_init_table(document);
|
||||
}
|
||||
_add_doc(document);
|
||||
}
|
||||
timer.stop();
|
||||
bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
|
||||
timer.start();
|
||||
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||
timer.stop();
|
||||
bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
|
||||
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||
return;
|
||||
myFireEvent(dp);
|
||||
}
|
||||
private void _read_mogodb() {
|
||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||
timer.start();
|
||||
// Access the collection
|
||||
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
||||
|
||||
private void _init_table(BsonDocument doc_row) {
|
||||
bts.Log.assert(_table == null);
|
||||
var schema = BlueDataUtility.form_schema($"{_database}_{_table}",
|
||||
Tuple.Create("id", TypeCode.Int32),
|
||||
Tuple.Create("doc_id", TypeCode.String),
|
||||
Tuple.Create("key", TypeCode.String),
|
||||
Tuple.Create("value", TypeCode.String));
|
||||
_table = new BlueTable(schema);
|
||||
}
|
||||
// Fetch all documents from the collection
|
||||
var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
||||
_table = _table == null ? null : new BlueTable(_table.schema);
|
||||
// Iterate through the documents and print values
|
||||
foreach (var document in documents.ToEnumerable()) {
|
||||
bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
||||
if (_table == null) {
|
||||
_init_table(document);
|
||||
}
|
||||
_add_doc(document);
|
||||
}
|
||||
timer.stop();
|
||||
bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
|
||||
timer.start();
|
||||
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||
timer.stop();
|
||||
bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
|
||||
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||
return;
|
||||
myFireEvent(dp);
|
||||
}
|
||||
|
||||
private void _add_doc(BsonDocument doc) {
|
||||
var doc_id_bson = doc["_id"];
|
||||
var doc_id = doc_id_bson.ToString();
|
||||
Func<string, string, int> get_id = (string doc_id, string key) => {
|
||||
string hash_key = $"{doc_id}_{key}";
|
||||
int id = 0;
|
||||
if (!_id_map.TryGetValue(hash_key, out id)) {
|
||||
id = _id_map.Count;
|
||||
_id_map.Add(hash_key, id);
|
||||
}
|
||||
return id;
|
||||
};
|
||||
foreach (var n in doc.Names) {
|
||||
string key = n;
|
||||
var id = get_id(doc_id, key);
|
||||
MutableBlueRow nr = new MutableBlueRow(_table.schema);
|
||||
var val = doc[n].ToString();
|
||||
nr.setValue("id", id);
|
||||
nr.setValue("doc_id", doc_id);
|
||||
nr.setValue("key", key);
|
||||
nr.setValue("value", val);
|
||||
_table.addRow(nr);
|
||||
}
|
||||
}
|
||||
}
|
||||
private void _init_table(BsonDocument doc_row) {
|
||||
bts.Log.assert(_table == null);
|
||||
var schema = BlueDataUtility.form_schema($"{_database}_{_table}",
|
||||
Tuple.Create("id", TypeCode.Int32),
|
||||
Tuple.Create("doc_id", TypeCode.String),
|
||||
Tuple.Create("key", TypeCode.String),
|
||||
Tuple.Create("value", TypeCode.String));
|
||||
_table = new BlueTable(schema);
|
||||
}
|
||||
|
||||
}
|
||||
private void _add_doc(BsonDocument doc) {
|
||||
var doc_id_bson = doc["_id"];
|
||||
var doc_id = doc_id_bson.ToString();
|
||||
Func<string, string, int> get_id = (string doc_id, string key) => {
|
||||
string hash_key = $"{doc_id}_{key}";
|
||||
int id = 0;
|
||||
if (!_id_map.TryGetValue(hash_key, out id)) {
|
||||
id = _id_map.Count;
|
||||
_id_map.Add(hash_key, id);
|
||||
}
|
||||
return id;
|
||||
};
|
||||
foreach (var n in doc.Names) {
|
||||
string key = n;
|
||||
var id = get_id(doc_id, key);
|
||||
MutableBlueRow nr = new MutableBlueRow(_table.schema);
|
||||
var val = doc[n].ToString();
|
||||
nr.setValue("id", id);
|
||||
nr.setValue("doc_id", doc_id);
|
||||
nr.setValue("key", key);
|
||||
nr.setValue("value", val);
|
||||
_table.addRow(nr);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user