195 lines
7.0 KiB
C#
195 lines
7.0 KiB
C#
namespace Bts.DataStream;
|
|
|
|
using BCG;
|
|
|
|
using bts;
|
|
|
|
using Bts.DataStreamCore;
|
|
|
|
using mbts;
|
|
|
|
using MongoDB.Bson;
|
|
using MongoDB.Driver;
|
|
|
|
using Spark.Core.Disposables;
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Diagnostics.CodeAnalysis;
|
|
using System.Reactive.Subjects;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
|
|
public class MongoCollectionStream : ViewStream, IObservable<DataPacket> {
|
|
readonly string collectionName;
|
|
readonly string databaseName;
|
|
readonly string url;
|
|
readonly object syncRoot = new();
|
|
readonly CancellationTokenSource cts = new();
|
|
readonly Subject<DataPacket> subject = new();
|
|
|
|
bool hasStarted = false;
|
|
IBlueRowFactory<BsonDocument>? rowFactory;
|
|
MongoClient? clientBacking;
|
|
IMongoDatabase? databaseBacking;
|
|
IMongoCollection<BsonDocument>? collectionBacking = null;
|
|
BlueTableImmutable? table = null;
|
|
|
|
public MongoCollectionStream(string collectionName)
|
|
: this(
|
|
collectionName,
|
|
Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
|
|
Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
|
|
}
|
|
|
|
public MongoCollectionStream(string collectionName, string databaseName)
|
|
: this(
|
|
collectionName,
|
|
databaseName,
|
|
Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
|
|
}
|
|
|
|
public MongoCollectionStream(string collectionName, string databaseName, string url)
|
|
: this(collectionName, databaseName, url, null) { }
|
|
|
|
internal MongoCollectionStream(
|
|
string collectionName, string databaseName, string url, IBlueRowFactory<BsonDocument>? rowFactory) {
|
|
this.collectionName = collectionName;
|
|
this.databaseName = databaseName;
|
|
this.url = url;
|
|
this.rowFactory = rowFactory;
|
|
mySetRealSource(new ObservableViewStream(this));
|
|
}
|
|
|
|
MongoClient client => clientBacking ??= new(url);
|
|
IMongoDatabase database => databaseBacking ??= client.GetDatabase(databaseName);
|
|
IMongoCollection<BsonDocument>? collection {
|
|
get => Volatile.Read(ref collectionBacking);
|
|
set => Volatile.Write(ref collectionBacking, value);
|
|
}
|
|
|
|
public IDisposable Subscribe(IObserver<DataPacket> observer) {
|
|
lock (syncRoot) {
|
|
if (hasStarted) {
|
|
if (table is not null) {
|
|
// The first update has already been fired. Seed observer w/ current state.
|
|
observer.OnNext(new DataPacket(table, table, new BlueTable(table.schema)));
|
|
}
|
|
return subject.Subscribe(observer);
|
|
}
|
|
hasStarted = true;
|
|
}
|
|
var subscription = subject.Subscribe(observer);
|
|
// Retrieve collection snapshot to bootstrap sequence
|
|
initTable().ignore();
|
|
// Watch changes
|
|
Task.Run(watchChanges);
|
|
return Disposable.Create(Dispose);
|
|
|
|
void Dispose() {
|
|
Log.Gen.info("Disposing MongoDbChangeObservable subscription");
|
|
subscription.Dispose();
|
|
cts.Cancel();
|
|
cts.Dispose();
|
|
}
|
|
}
|
|
|
|
protected virtual IBlueRowFactory<BsonDocument> getRowFactory(IEnumerable<BsonDocument> docs) =>
|
|
MongoSchemaAdaptor.InferSchema(docs);
|
|
|
|
async Task watchChanges() {
|
|
Log.Gen.info(
|
|
$"MongoCollectionStream. Watching changes. thread: {Environment.CurrentManagedThreadId}");
|
|
while (collection is null) {
|
|
await Task.Delay(100);
|
|
}
|
|
Log.notnull(collection);
|
|
using var cursor = collection.Watch(new ChangeStreamOptions {
|
|
FullDocument = ChangeStreamFullDocumentOption.UpdateLookup
|
|
});
|
|
try {
|
|
while (await cursor.MoveNextAsync(cts.Token)) {
|
|
/// Ignore while waiting for init.
|
|
lock (syncRoot) {
|
|
/// Note: <see cref="rowFactory"/> and <see cref="table"/> are initialized together
|
|
/// in the <see cref="initTable"/> fn since both depend on the schema, which is
|
|
/// inferred from the first incoming BsonDocument(s).
|
|
if (table is null || rowFactory is null) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
var bd = table.ToDataPacketBuilder();
|
|
foreach (var ch in cursor.Current) {
|
|
var opDescJson = ch.OperationDescription?.ToJson() ?? string.Empty;
|
|
var updDescJson = ch.UpdateDescription?.ToJson() ?? string.Empty;
|
|
Log.Gen.info(
|
|
$"MongoCollectionStream.watchChanges. Doc changed. type: {ch.OperationType} " +
|
|
$"changes:\n\t\t{updDescJson}");
|
|
switch (ch.OperationType) {
|
|
case ChangeStreamOperationType.Insert: {
|
|
bd.AddOrUpdate(Row());
|
|
break;
|
|
}
|
|
case ChangeStreamOperationType.Update: {
|
|
bd.Remove(OldRow().Id());
|
|
bd.AddOrUpdate(Row());
|
|
break;
|
|
}
|
|
case ChangeStreamOperationType.Replace: {
|
|
bd.Remove(OldRow().Id());
|
|
bd.AddOrUpdate(Row());
|
|
break;
|
|
}
|
|
case ChangeStreamOperationType.Delete: {
|
|
bd.Remove(OldRow().Id());
|
|
break;
|
|
}
|
|
default:
|
|
Log.assert(false, $"Unsupported op type '{ch.OperationType}'");
|
|
break;
|
|
}
|
|
|
|
IBlueRow Row() =>
|
|
rowFactory.GetBlueRow(ch.FullDocument);
|
|
|
|
IBlueRow OldRow() =>
|
|
rowFactory.GetBlueRow(ch.FullDocumentBeforeChange);
|
|
}
|
|
var dp = bd.ToDataPacket();
|
|
if (!dp.HasChanges()) {
|
|
Log.Gen.warn(
|
|
$"MongoCollectionStream. Ignoring update because it didn't change any rows.");
|
|
continue;
|
|
}
|
|
lock (syncRoot) {
|
|
table = (BlueTableImmutable)dp.newRows;
|
|
subject.OnNext(dp);
|
|
}
|
|
}
|
|
} catch (TaskCanceledException ex) {
|
|
Log.Gen.warn(
|
|
$"MongoCollectionStream. 'Watch' task cancelled (this is expected if subscription was disposed). {ex}");
|
|
}
|
|
}
|
|
|
|
|
|
[MemberNotNull(nameof(collection))]
|
|
async Task initTable() {
|
|
collection = database.GetCollection<BsonDocument>(collectionName);
|
|
var cursor = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
|
var docList = await cursor.ToListAsync();
|
|
rowFactory ??= getRowFactory(docList);
|
|
var firstTable = new BlueTableImmutable(rowFactory.Schema).ToDataPacketBuilder();
|
|
foreach (var doc in docList) {
|
|
var row = rowFactory.GetBlueRow(doc);
|
|
firstTable.AddOrUpdate(row);
|
|
}
|
|
var dp = firstTable.ToDataPacket();
|
|
lock (syncRoot) {
|
|
table = (BlueTableImmutable)dp.newRows;
|
|
subject.OnNext(dp);
|
|
}
|
|
}
|
|
}
|