namespace Bts.DataStream; using BCG; using bts; using Bts.DataStreamCore; using mbts; using MongoDB.Bson; using MongoDB.Driver; using Spark.Core.Disposables; using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Reactive.Subjects; using System.Threading; using System.Threading.Tasks; public class MongoCollectionStream : ViewStream, IObservable { readonly string collectionName; readonly string databaseName; readonly string url; readonly object syncRoot = new(); readonly CancellationTokenSource cts = new(); readonly Subject subject = new(); bool hasStarted = false; IBlueRowFactory? rowFactory; MongoClient? clientBacking; IMongoDatabase? databaseBacking; IMongoCollection? collectionBacking = null; BlueTableImmutable? table = null; public MongoCollectionStream(string collectionName) : this( collectionName, Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty, Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) { } public MongoCollectionStream(string collectionName, string databaseName) : this( collectionName, databaseName, Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) { } public MongoCollectionStream(string collectionName, string databaseName, string url) : this(collectionName, databaseName, url, null) { } internal MongoCollectionStream( string collectionName, string databaseName, string url, IBlueRowFactory? rowFactory) { this.collectionName = collectionName; this.databaseName = databaseName; this.url = url; this.rowFactory = rowFactory; mySetRealSource(new ObservableViewStream(this)); } MongoClient client => clientBacking ??= new(url); IMongoDatabase database => databaseBacking ??= client.GetDatabase(databaseName); IMongoCollection? collection { get => Volatile.Read(ref collectionBacking); set => Volatile.Write(ref collectionBacking, value); } public IDisposable Subscribe(IObserver observer) { lock (syncRoot) { if (hasStarted) { if (table is not null) { // The first update has already been fired. Seed observer w/ current state. observer.OnNext(new DataPacket(table, table, new BlueTable(table.schema))); } return subject.Subscribe(observer); } hasStarted = true; } var subscription = subject.Subscribe(observer); // Retrieve collection snapshot to bootstrap sequence initTable().ignore(); // Watch changes Task.Run(watchChanges); return Disposable.Create(Dispose); void Dispose() { Log.Gen.info("Disposing MongoDbChangeObservable subscription"); subscription.Dispose(); cts.Cancel(); cts.Dispose(); } } protected virtual IBlueRowFactory getRowFactory(IEnumerable docs) => MongoSchemaAdaptor.InferSchema(docs); async Task watchChanges() { Log.Gen.info( $"MongoCollectionStream. Watching changes. thread: {Environment.CurrentManagedThreadId}"); while (collection is null) { await Task.Delay(100); } Log.notnull(collection); using var cursor = collection.Watch(new ChangeStreamOptions { FullDocument = ChangeStreamFullDocumentOption.UpdateLookup }); try { while (await cursor.MoveNextAsync(cts.Token)) { /// Ignore while waiting for init. lock (syncRoot) { /// Note: and are initialized together /// in the fn since both depend on the schema, which is /// inferred from the first incoming BsonDocument(s). if (table is null || rowFactory is null) { continue; } } var bd = table.ToDataPacketBuilder(); foreach (var ch in cursor.Current) { var opDescJson = ch.OperationDescription?.ToJson() ?? string.Empty; var updDescJson = ch.UpdateDescription?.ToJson() ?? string.Empty; Log.Gen.info( $"MongoCollectionStream.watchChanges. Doc changed. type: {ch.OperationType} " + $"changes:\n\t\t{updDescJson}"); switch (ch.OperationType) { case ChangeStreamOperationType.Insert: { bd.AddOrUpdate(Row()); break; } case ChangeStreamOperationType.Update: { bd.Remove(OldRow().Id()); bd.AddOrUpdate(Row()); break; } case ChangeStreamOperationType.Replace: { bd.Remove(OldRow().Id()); bd.AddOrUpdate(Row()); break; } case ChangeStreamOperationType.Delete: { bd.Remove(OldRow().Id()); break; } default: Log.assert(false, $"Unsupported op type '{ch.OperationType}'"); break; } IBlueRow Row() => rowFactory.GetBlueRow(ch.FullDocument); IBlueRow OldRow() => rowFactory.GetBlueRow(ch.FullDocumentBeforeChange); } var dp = bd.ToDataPacket(); if (!dp.HasChanges()) { Log.Gen.warn( $"MongoCollectionStream. Ignoring update because it didn't change any rows."); continue; } lock (syncRoot) { table = (BlueTableImmutable)dp.newRows; subject.OnNext(dp); } } } catch (TaskCanceledException ex) { Log.Gen.warn( $"MongoCollectionStream. 'Watch' task cancelled (this is expected if subscription was disposed). {ex}"); } } [MemberNotNull(nameof(collection))] async Task initTable() { collection = database.GetCollection(collectionName); var cursor = collection.Find(Builders.Filter.Empty).ToCursor(); var docList = await cursor.ToListAsync(); rowFactory ??= getRowFactory(docList); var firstTable = new BlueTableImmutable(rowFactory.Schema).ToDataPacketBuilder(); foreach (var doc in docList) { var row = rowFactory.GetBlueRow(doc); firstTable.AddOrUpdate(row); } var dp = firstTable.ToDataPacket(); lock (syncRoot) { table = (BlueTableImmutable)dp.newRows; subject.OnNext(dp); } } }