Files
spark/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoCollectionStream.cs
T

195 lines
7.0 KiB
C#

namespace Bts.DataStream;
using BCG;
using bts;
using Bts.DataStreamCore;
using mbts;
using MongoDB.Bson;
using MongoDB.Driver;
using Spark.Core.Disposables;
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Reactive.Subjects;
using System.Threading;
using System.Threading.Tasks;
public class MongoCollectionStream : ViewStream, IObservable<DataPacket> {
readonly string collectionName;
readonly string databaseName;
readonly string url;
readonly object syncRoot = new();
readonly CancellationTokenSource cts = new();
readonly Subject<DataPacket> subject = new();
bool hasStarted = false;
IBlueRowFactory<BsonDocument>? rowFactory;
MongoClient? clientBacking;
IMongoDatabase? databaseBacking;
IMongoCollection<BsonDocument>? collectionBacking = null;
BlueTableImmutable? table = null;
public MongoCollectionStream(string collectionName)
: this(
collectionName,
Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
}
public MongoCollectionStream(string collectionName, string databaseName)
: this(
collectionName,
databaseName,
Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
}
public MongoCollectionStream(string collectionName, string databaseName, string url)
: this(collectionName, databaseName, url, null) { }
internal MongoCollectionStream(
string collectionName, string databaseName, string url, IBlueRowFactory<BsonDocument>? rowFactory) {
this.collectionName = collectionName;
this.databaseName = databaseName;
this.url = url;
this.rowFactory = rowFactory;
mySetRealSource(new ObservableViewStream(this));
}
MongoClient client => clientBacking ??= new(url);
IMongoDatabase database => databaseBacking ??= client.GetDatabase(databaseName);
IMongoCollection<BsonDocument>? collection {
get => Volatile.Read(ref collectionBacking);
set => Volatile.Write(ref collectionBacking, value);
}
public IDisposable Subscribe(IObserver<DataPacket> observer) {
lock (syncRoot) {
if (hasStarted) {
if (table is not null) {
// The first update has already been fired. Seed observer w/ current state.
observer.OnNext(new DataPacket(table, table, new BlueTable(table.schema)));
}
return subject.Subscribe(observer);
}
hasStarted = true;
}
var subscription = subject.Subscribe(observer);
// Retrieve collection snapshot to bootstrap sequence
initTable().ignore();
// Watch changes
Task.Run(watchChanges);
return Disposable.Create(Dispose);
void Dispose() {
Log.Gen.info("Disposing MongoDbChangeObservable subscription");
subscription.Dispose();
cts.Cancel();
cts.Dispose();
}
}
protected virtual IBlueRowFactory<BsonDocument> getRowFactory(IEnumerable<BsonDocument> docs) =>
MongoSchemaAdaptor.InferSchema(docs);
async Task watchChanges() {
Log.Gen.info(
$"MongoCollectionStream. Watching changes. thread: {Environment.CurrentManagedThreadId}");
while (collection is null) {
await Task.Delay(100);
}
Log.notnull(collection);
using var cursor = collection.Watch(new ChangeStreamOptions {
FullDocument = ChangeStreamFullDocumentOption.UpdateLookup
});
try {
while (await cursor.MoveNextAsync(cts.Token)) {
/// Ignore while waiting for init.
lock (syncRoot) {
/// Note: <see cref="rowFactory"/> and <see cref="table"/> are initialized together
/// in the <see cref="initTable"/> fn since both depend on the schema, which is
/// inferred from the first incoming BsonDocument(s).
if (table is null || rowFactory is null) {
continue;
}
}
var bd = table.ToDataPacketBuilder();
foreach (var ch in cursor.Current) {
var opDescJson = ch.OperationDescription?.ToJson() ?? string.Empty;
var updDescJson = ch.UpdateDescription?.ToJson() ?? string.Empty;
Log.Gen.info(
$"MongoCollectionStream.watchChanges. Doc changed. type: {ch.OperationType} " +
$"changes:\n\t\t{updDescJson}");
switch (ch.OperationType) {
case ChangeStreamOperationType.Insert: {
bd.AddOrUpdate(Row());
break;
}
case ChangeStreamOperationType.Update: {
bd.Remove(OldRow().Id());
bd.AddOrUpdate(Row());
break;
}
case ChangeStreamOperationType.Replace: {
bd.Remove(OldRow().Id());
bd.AddOrUpdate(Row());
break;
}
case ChangeStreamOperationType.Delete: {
bd.Remove(OldRow().Id());
break;
}
default:
Log.assert(false, $"Unsupported op type '{ch.OperationType}'");
break;
}
IBlueRow Row() =>
rowFactory.GetBlueRow(ch.FullDocument);
IBlueRow OldRow() =>
rowFactory.GetBlueRow(ch.FullDocumentBeforeChange);
}
var dp = bd.ToDataPacket();
if (!dp.HasChanges()) {
Log.Gen.warn(
$"MongoCollectionStream. Ignoring update because it didn't change any rows.");
continue;
}
lock (syncRoot) {
table = (BlueTableImmutable)dp.newRows;
subject.OnNext(dp);
}
}
} catch (TaskCanceledException ex) {
Log.Gen.warn(
$"MongoCollectionStream. 'Watch' task cancelled (this is expected if subscription was disposed). {ex}");
}
}
[MemberNotNull(nameof(collection))]
async Task initTable() {
collection = database.GetCollection<BsonDocument>(collectionName);
var cursor = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
var docList = await cursor.ToListAsync();
rowFactory ??= getRowFactory(docList);
var firstTable = new BlueTableImmutable(rowFactory.Schema).ToDataPacketBuilder();
foreach (var doc in docList) {
var row = rowFactory.GetBlueRow(doc);
firstTable.AddOrUpdate(row);
}
var dp = firstTable.ToDataPacket();
lock (syncRoot) {
table = (BlueTableImmutable)dp.newRows;
subject.OnNext(dp);
}
}
}