From 15f811ebfbba28faa315158c50dd806771bcb4b7 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 4 Mar 2024 11:11:38 -0500 Subject: [PATCH] feat: Add IncrementalMongoDbStream --- .../DataStream/ObservableViewStream.cs | 7 +- .../MongoDB/IncrementalMongoDbStream.cs | 35 +++ .../MongoDB/MongoDbChangeObservable.cs | 208 ++++++++++++++++++ .../Primitives/MongoDB/MongoDbExtensions.cs | 39 ++++ .../BlueData/BlueTableImmutable.cs | 80 +++++++ .../BlueData/BlueTableSchemaAdaptor.cs | 84 +++++++ .../DataStreamCore/BlueRowExtensions.cs | 11 + 7 files changed, 463 insertions(+), 1 deletion(-) create mode 100644 src/DataStreamSuite/DataStream/Primitives/MongoDB/IncrementalMongoDbStream.cs create mode 100644 src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbChangeObservable.cs create mode 100644 src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbExtensions.cs create mode 100644 src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs create mode 100644 src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdaptor.cs create mode 100644 src/DataStreamSuite/DataStreamCore/BlueRowExtensions.cs diff --git a/src/DataStreamSuite/DataStream/ObservableViewStream.cs b/src/DataStreamSuite/DataStream/ObservableViewStream.cs index f4e98b035..8c454aaa0 100644 --- a/src/DataStreamSuite/DataStream/ObservableViewStream.cs +++ b/src/DataStreamSuite/DataStream/ObservableViewStream.cs @@ -6,13 +6,18 @@ using bts; using System; -class ObservableViewStream : ViewStream, IObservable { +public class ObservableViewStream : ViewStream, IObservable, IDisposable { readonly IObservable source; IDisposable? subscription; public ObservableViewStream(IObservable source) => this.source = source; + public void Dispose() { + subscription?.Dispose(); + subscription = null; + } + protected override void myOnFirstSubscriber() { Log.isnull(subscription); subscription = source.Subscribe(myFireEvent); diff --git a/src/DataStreamSuite/DataStream/Primitives/MongoDB/IncrementalMongoDbStream.cs b/src/DataStreamSuite/DataStream/Primitives/MongoDB/IncrementalMongoDbStream.cs new file mode 100644 index 000000000..6a0f8ce60 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/MongoDB/IncrementalMongoDbStream.cs @@ -0,0 +1,35 @@ +namespace Bts.DataStream; + +using BCG; + +using System; +using System.Collections.Generic; + +public sealed class IncrementalMongoDbStream : ViewStream { + readonly string collectionName; + readonly string databaseName; + readonly string url; + readonly string index; + readonly Dictionary idMap = []; + readonly Dictionary columnMap = []; + readonly MongoDbChangeStreamObservable source; + + public IncrementalMongoDbStream() + : this( + Environment.GetEnvironmentVariable("MONGO_DB_COLL") ?? string.Empty, + Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty, + Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) { + } + + public IncrementalMongoDbStream(string collectionName, string databaseName, string url) + : this(collectionName, databaseName, url, "_id") { } + + public IncrementalMongoDbStream(string collectionName, string databaseName, string url, string index) { + this.collectionName = collectionName; + this.databaseName = databaseName; + this.url = url; + this.index = index; + source = new(collectionName, databaseName, url); + mySetRealSource(source.AsViewStream()); + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbChangeObservable.cs b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbChangeObservable.cs new file mode 100644 index 000000000..16a351d7a --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbChangeObservable.cs @@ -0,0 +1,208 @@ +namespace Bts.DataStream; + +using BCG; + +using bts; +using bts.collections; + +using Bts.DataStreamCore; + +using mbts; + +using MongoDB.Bson; +using MongoDB.Driver; + +using Spark.Core; +using Spark.Core.Disposables; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive.Subjects; +using System.Threading; +using System.Threading.Tasks; + +public class MongoDbChangeStreamObservable : IObservable { + readonly string collectionName; + readonly string databaseName; + readonly string url; + readonly object locker = new(); + readonly CancellationTokenSource cts = new(); + readonly BlueTableSchemaAdaptor schemaBd = new("MongoDbChangeStream"); + + bool hasStarted = false; + MongoClient? clientBacking; + IMongoDatabase? databaseBacking; + IMongoCollection? collectionBacking = null; + BlueTableImmutable? table = null; + Subject subject = new(); + + public MongoDbChangeStreamObservable(string collectionName, string databaseName, string url) { + this.collectionName = collectionName; + this.databaseName = databaseName; + this.url = url; + } + + MongoClient client => clientBacking ??= new(url); + IMongoDatabase database => databaseBacking ??= client.GetDatabase(databaseName); + IMongoCollection? collection { + get => Volatile.Read(ref collectionBacking); + set => Volatile.Write(ref collectionBacking, value); + } + + public IDisposable Subscribe(IObserver observer) { + lock (locker) { + if (hasStarted) { + if (table is not null) { + // The first update has already been fired. Seed observer w/ current state. + observer.OnNext(new DataPacket(table, table, new BlueTable(table.schema))); + } + return subject.Subscribe(observer); + } + hasStarted = true; + } + var subscription = subject.Subscribe(observer); + // Retrieve collection snapshot in background + initTable().ignore(); + // Listen for changes in background + Listen().ignore(); + return Disposable.Create(Dispose); + + void Dispose() { + Log.Gen.info("Disposing MongoDbChangeObservable subscription"); + subscription.Dispose(); + cts.Cancel(); + cts.Dispose(); + } + + async Task Listen() { + while (collection is null) { + Thread.Yield(); + } + using var cursor = collection.Watch(new ChangeStreamOptions { + FullDocument = ChangeStreamFullDocumentOption.UpdateLookup + }); + try { + while (await cursor.MoveNextAsync(cts.Token)) { + // Ignore changes while waiting for collection snapshot + lock (locker) { + if (table is null) { + continue; + } + } + var bd = table.ToDataPacketBuilder(); + foreach (var ch in cursor.Current) { + var opDescJson = ch.OperationDescription?.ToJson() ?? string.Empty; + var updDescJson = ch.UpdateDescription?.ToJson() ?? string.Empty; + var doc = ch.FullDocument; + var docJson = doc.ToJson(); + Log.Gen.info( + $"IncrementalMongoDbStream doc changed. type: {ch.OperationType} " + + $"\n\tchanges:\n\t\t{updDescJson}\n\tnewValue:\n\t\t{docJson}"); + switch (ch.OperationType) { + case ChangeStreamOperationType.Insert: { + Log.Gen.info( + $"IncrementalMongoDbStream doc changed. type: {ch.OperationType} " + + $"\n\tchanges:\n\t\t{updDescJson}\n\tnewValue:\n\t\t{docJson}"); + var row = schemaBd.GetBlueRow(doc); + bd.AddOrUpdate(row); + break; + } + case ChangeStreamOperationType.Update: { + var oldRow = schemaBd.GetBlueRow(ch.FullDocumentBeforeChange); + bd.Remove(oldRow.Id()); + var newRow = schemaBd.GetBlueRow(ch.FullDocument); + bd.AddOrUpdate(newRow); + break; + } + case ChangeStreamOperationType.Replace: { + var oldRow = schemaBd.GetBlueRow(ch.FullDocumentBeforeChange); + bd.Remove(oldRow.Id()); + var newRow = schemaBd.GetBlueRow(ch.FullDocument); + bd.AddOrUpdate(newRow); + break; + } + default: + Log.assert(false, $"Unsupported op type '{ch.OperationType}'"); + break; + } + } + var dp = bd.ToDataPacket(); + if (!dp.HasChanges()) { + Log.Gen.warn( + $"MongoDbChangeObservable Ignoring update because it didn't change any rows."); + continue; + } + lock (locker) { + table = (BlueTableImmutable)dp.newRows; + subject.OnNext(dp); + } + } + } catch (TaskCanceledException ex) { + Log.Gen.warn( + $"MongoDbChangeObservable 'Watch' task cancelled (this is " + + $"expected if subscription was disposed). {ex}"); + } + } + } + + async Task initTable() { + collection = database.GetCollection(collectionName); + var cursor = collection.Find(Builders.Filter.Empty).ToCursor(); + var docList = await cursor.ToListAsync(); + initSchema(docList); + var firstTable = new BlueTableImmutable(schemaBd.Schema).ToDataPacketBuilder(); + foreach (var doc in docList) { + var row = schemaBd.GetBlueRow(doc); + firstTable.AddOrUpdate(row); + } + var dp = firstTable.ToDataPacket(); + lock (locker) { + table = (BlueTableImmutable)dp.newRows; + subject.OnNext(dp); + } + } + + void initSchema(IEnumerable docs) { + foreach (var doc in docs) { + foreach (var el in doc) { + if (schemaBd.Contains(el.Name)) { + continue; + } + switch (el.Value.BsonType) { + case BsonType.Array: + schemaBd.AddColumn(el.Name, d => d.GetArrayOrDefault(el.Name)?.ToString() ?? string.Empty); + break; + case BsonType.String: + schemaBd.AddColumn(el.Name, d => d.GetStringOrDefault(el.Name) ?? string.Empty); + break; + case BsonType.Int32: + schemaBd.AddColumn(el.Name, d => d.GetInt32OrDefault(el.Name)); + break; + case BsonType.Int64: + schemaBd.AddColumn(el.Name, d => d.GetInt64OrDefault(el.Name)); + break; + case BsonType.Double: + schemaBd.AddColumn(el.Name, d => d.GetDoubleOrDefault(el.Name)); + break; + case BsonType.DateTime: + schemaBd.AddColumn(el.Name, d => d.GetDateTimeOrDefault(el.Name)); + break; + case BsonType.Timestamp: + schemaBd.AddColumn(el.Name, d => d.GetTimestampOrDefault(el.Name)); + break; + case BsonType.ObjectId: + schemaBd.AddColumn(el.Name, d => d.GetObjectIdOrDefault(el.Name)?.ToString()); + break; + case BsonType.Null: + Log.Gen.warn( + $"Ignoring column '{el.Name}' because at least one element is null"); + break; + default: + throw new InvalidOperationException( + $"Unsupported BsonType '{el.Value.BsonType}'"); + } + } + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbExtensions.cs b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbExtensions.cs new file mode 100644 index 000000000..a15f5bfc0 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbExtensions.cs @@ -0,0 +1,39 @@ +namespace Bts.DataStream; + +using MongoDB.Bson; + +using System; + +public static class MongoDbExtensions { + public static BsonArray? GetArrayOrDefault(this BsonDocument doc, string elementName) => + doc.TryGetValue(elementName, out var val) && val.IsBsonArray ? + val.AsBsonArray : null; + + public static string? GetStringOrDefault(this BsonDocument doc, string elementName) => + doc.TryGetValue(elementName, out var val) && val.IsString ? + val.AsString : null; + + public static int? GetInt32OrDefault(this BsonDocument doc, string elementName) => + doc.TryGetValue(elementName, out var val) && val.IsInt32 ? + val.AsInt32 : null; + + public static long? GetInt64OrDefault(this BsonDocument doc, string elementName) => + doc.TryGetValue(elementName, out var val) && val.IsInt64 ? + val.AsInt64 : null; + + public static double? GetDoubleOrDefault(this BsonDocument doc, string elementName) => + doc.TryGetValue(elementName, out var val) && val.IsDouble ? + val.AsDouble : null; + + public static DateTime? GetDateTimeOrDefault(this BsonDocument doc, string elementName) => + doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ? + val.ToUniversalTime() : null; + + public static DateTime? GetTimestampOrDefault(this BsonDocument doc, string elementName) => + doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ? + val.ToUniversalTime() : null; + + public static ObjectId? GetObjectIdOrDefault(this BsonDocument doc, string elementName) => + doc.TryGetValue(elementName, out var val) && val.IsObjectId ? + val.AsObjectId : null; +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs new file mode 100644 index 000000000..a287faec8 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs @@ -0,0 +1,80 @@ +namespace Bts.DataStreamCore; + +using BCG; + +using System; +using System.Collections; +using System.Collections.Immutable; + +// I would have named this ImmutableBlueTable, but that was already taken. +public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection { + readonly ImmutableDictionary rows; + + public BlueTableImmutable(IBlueTableSchema schema) + : this(schema, ImmutableDictionary.Empty) { } + + public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary rows) { + this.schema = schema; + this.rows = rows; + } + + public int Count => rows.Count; + public bool IsSynchronized => false; + public IBlueTableSchema schema { get; } + public object SyncRoot => this; + + public IBlueRow this[int id] => rows[id]; + + public bool containsID(int id) => + rows.ContainsKey(id); + + public void CopyTo(Array array, int index) { + foreach (var r in rows.Values) { + array.SetValue(r, index++); + } + } + + public IEnumerator GetEnumerator() => + rows.Values.GetEnumerator(); + + public DataPacketBuilder ToDataPacketBuilder() => + new(this); + + public class DataPacketBuilder { + readonly IBlueTableSchema schema; + readonly ImmutableDictionary.Builder table; + readonly ImmutableDictionary.Builder added = + ImmutableDictionary.Empty.ToBuilder(); + readonly ImmutableDictionary.Builder removed = + ImmutableDictionary.Empty.ToBuilder(); + + public DataPacketBuilder(BlueTableImmutable table) { + schema = table.schema; + this.table = table.rows.ToBuilder(); + } + + public void AddOrUpdate(IBlueRow row) { + var id = row.Id(); + added[id] = row; + if (table.TryGetValue(id, out var orig)) { + removed.Remove(id); + } + } + + public void Remove(int id) { + added.Remove(id); + if (table.TryGetValue(id, out var orig)) { + removed[id] = orig; + } + } + + public DataPacket ToDataPacket() { + table.RemoveRange(removed.Keys); + table.AddRange(added); + var newRows = new BlueTableImmutable(schema, table.ToImmutable()); + var addedRows = new BlueTableImmutable(schema, added.ToImmutable()); + var removedRows = new BlueTableImmutable(schema, removed.ToImmutable()); + return new(newRows, addedRows, removedRows); + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdaptor.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdaptor.cs new file mode 100644 index 000000000..cebed4fa8 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdaptor.cs @@ -0,0 +1,84 @@ +#nullable enable +namespace Bts.DataStream; + +using BCG; + +using bts.collections; + +using System; +using System.Collections.Generic; +using System.Linq; + +public class BlueTableSchemaAdaptor { + readonly string name; + readonly List columns = []; + BlueTableSchema? schema; + + public BlueTableSchemaAdaptor(string name) => + this.name = name; + + public IBlueTableSchema Schema => + schema ??= new BlueTableSchema([.. columns.WrapSelectorRO(c => c.Info)], name); + + public bool Contains(string name) => + columns.Any(c => c.Info.colName == name); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r), typeof(int)); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(int)); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r), typeof(long)); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(long)); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r), typeof(double)); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(double)); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r), typeof(DateTime)); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(DateTime)); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(string)); + + public void AddColumn(string name, Func getValue, Type dataType) { + if (!ColumnInfo.isAllowedType(dataType)) { + throw new ArgumentException( + $"'{dataType}' is not a supported column type", nameof(dataType)); + } + // Invalidate cached schema (set it to null) + schema = null; + columns.Add(new ColumnAdaptor(new(name, columns.Count, dataType), r => getValue(r))); + } + + public IBlueRow GetBlueRow(T item) { + var mbr = new MutableBlueRow(Schema); + foreach (var col in columns) { + mbr.setValue(col.Info.colNo, col.GetValue(item)); + } + return mbr; + } + + class ColumnAdaptor { + readonly Func getValue; + + public ColumnAdaptor(ColumnInfo info, Func getValue) { + Info = info; + this.getValue = getValue; + } + + public ColumnInfo Info { get; } + + public object? GetValue(T row) => + getValue(row); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueRowExtensions.cs b/src/DataStreamSuite/DataStreamCore/BlueRowExtensions.cs new file mode 100644 index 000000000..6d36052e0 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueRowExtensions.cs @@ -0,0 +1,11 @@ +namespace Bts.DataStreamCore; + +using BCG; + +using System; + +public static class BlueRowExtensions { + public static int Id(this IBlueRow row) => + (int)(row["id"] ?? throw new ArgumentException( + "Row doesn't have an 'id' column", nameof(row))); +}