From 97c82c486972abcc5fa563e6e2efafbebca3d4bc Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 4 Mar 2024 11:11:38 -0500 Subject: [PATCH] feat: Add [incremental] MongoCollectionStream --- .../MongoDB/MongoCollectionStream.cs | 203 ++++++++++++++++++ .../Primitives/MongoDB/MongoDbExtensions.cs | 63 ++++++ .../Primitives/MongoDB/MongoSchemaAdaptor.cs | 55 +++++ .../BlueData/BlueTableImmutable.cs | 80 +++++++ .../BlueData/BlueTableSchemaAdaptor.cs | 87 ++++++++ .../BlueData/IBlueRowFactory.cs | 9 + .../BlueData/IMutableBlueRowFactory.cs | 11 + .../DataStreamCore/BlueRowExtensions.cs | 11 + .../DataStreamCore/DataPacket.cs | 5 + 9 files changed, 524 insertions(+) create mode 100644 src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoCollectionStream.cs create mode 100644 src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbExtensions.cs create mode 100644 src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoSchemaAdaptor.cs create mode 100644 src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs create mode 100644 src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdaptor.cs create mode 100644 src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowFactory.cs create mode 100644 src/DataStreamSuite/DataStreamCore/BlueData/IMutableBlueRowFactory.cs create mode 100644 src/DataStreamSuite/DataStreamCore/BlueRowExtensions.cs diff --git a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoCollectionStream.cs b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoCollectionStream.cs new file mode 100644 index 000000000..fa1d5818f --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoCollectionStream.cs @@ -0,0 +1,203 @@ +namespace Bts.DataStream; + +using BCG; + +using bts; + +using Bts.DataStreamCore; + +using mbts; + +using MongoDB.Bson; +using MongoDB.Driver; + +using Spark.Core.Disposables; + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Reactive.Subjects; +using System.Threading; +using System.Threading.Tasks; + +public class MongoCollectionStream : ViewStream, IObservable { + readonly string collectionName; + readonly string databaseName; + readonly string url; + readonly object syncRoot = new(); + readonly CancellationTokenSource cts = new(); + readonly Subject subject = new(); + + bool hasStarted = false; + IBlueRowFactory? rowFactory; + MongoClient? clientBacking; + IMongoDatabase? databaseBacking; + IMongoCollection? collectionBacking = null; + BlueTableImmutable? table = null; + + public MongoCollectionStream() + : this( + Environment.GetEnvironmentVariable("MONGO_DB_COLL") ?? string.Empty, + Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty, + Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) { + } + + public MongoCollectionStream(string collectionName) + : this( + collectionName, + Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty, + Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) { + } + + public MongoCollectionStream(string collectionName, string databaseName) + : this( + collectionName, + databaseName, + Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) { + } + + public MongoCollectionStream(string collectionName, string databaseName, string url) + : this(collectionName, databaseName, url, null) { } + + internal MongoCollectionStream( + string collectionName, string databaseName, string url, IBlueRowFactory? rowFactory) { + this.collectionName = collectionName; + this.databaseName = databaseName; + this.url = url; + this.rowFactory = rowFactory; + mySetRealSource(new ObservableViewStream(this)); + } + + MongoClient client => clientBacking ??= new(url); + IMongoDatabase database => databaseBacking ??= client.GetDatabase(databaseName); + IMongoCollection? collection { + get => Volatile.Read(ref collectionBacking); + set => Volatile.Write(ref collectionBacking, value); + } + + public IDisposable Subscribe(IObserver observer) { + lock (syncRoot) { + if (hasStarted) { + if (table is not null) { + // The first update has already been fired. Seed observer w/ current state. + observer.OnNext(new DataPacket(table, table, new BlueTable(table.schema))); + } + return subject.Subscribe(observer); + } + hasStarted = true; + } + var subscription = subject.Subscribe(observer); + // Retrieve collection snapshot to bootstrap sequence + initTable().ignore(); + // Watch changes + Task.Run(watchChanges); + return Disposable.Create(Dispose); + + void Dispose() { + Log.Gen.info("Disposing MongoDbChangeObservable subscription"); + subscription.Dispose(); + cts.Cancel(); + cts.Dispose(); + } + } + + protected virtual IBlueRowFactory getRowFactory(IEnumerable docs) => + MongoSchemaAdaptor.InferSchema(docs); + + async Task watchChanges() { + Log.Gen.info( + $"MongoCollectionStreamSource. Watching changes. thread: {Environment.CurrentManagedThreadId}"); + while (collection is null) { + await Task.Delay(100); + } + Log.notnull(collection); + using var cursor = collection.Watch(new ChangeStreamOptions { + FullDocument = ChangeStreamFullDocumentOption.UpdateLookup + }); + try { + while (await cursor.MoveNextAsync(cts.Token)) { + // Ignore while waiting for init + lock (syncRoot) { + if (table is null) { + continue; + } + } + // schemaAdaptor is initialized after table; redundantly assert both + // are non-null to be safe, and clarify expectations. + Log.notnull(rowFactory); + Log.notnull(table); + + var bd = table.ToDataPacketBuilder(); + foreach (var ch in cursor.Current) { + var opDescJson = ch.OperationDescription?.ToJson() ?? string.Empty; + var updDescJson = ch.UpdateDescription?.ToJson() ?? string.Empty; + Log.Gen.info( + $"MongoCollectionStream.watchChanges. Doc changed. type: {ch.OperationType} " + + $"changes:\n\t\t{updDescJson}"); + switch (ch.OperationType) { + case ChangeStreamOperationType.Insert: { + bd.AddOrUpdate(Row()); + break; + } + case ChangeStreamOperationType.Update: { + bd.Remove(OldRow().Id()); + bd.AddOrUpdate(Row()); + break; + } + case ChangeStreamOperationType.Replace: { + bd.Remove(OldRow().Id()); + bd.AddOrUpdate(Row()); + break; + } + case ChangeStreamOperationType.Delete: { + bd.Remove(OldRow().Id()); + break; + } + default: + Log.assert(false, $"Unsupported op type '{ch.OperationType}'"); + break; + } + + IBlueRow Row() => + rowFactory.GetBlueRow(ch.FullDocumentBeforeChange); + + IBlueRow OldRow() => + rowFactory.GetBlueRow(ch.FullDocumentBeforeChange); + } + var dp = bd.ToDataPacket(); + if (!dp.HasChanges()) { + Log.Gen.warn( + $"MongoDbChangeObservable Ignoring update because it didn't change any rows."); + continue; + } + lock (syncRoot) { + table = (BlueTableImmutable)dp.newRows; + subject.OnNext(dp); + } + } + } catch (TaskCanceledException ex) { + Log.Gen.warn( + $"MongoDbChangeObservable 'Watch' task cancelled (this is " + + $"expected if subscription was disposed). {ex}"); + } + } + + + [MemberNotNull(nameof(collection))] + async Task initTable() { + collection = database.GetCollection(collectionName); + var cursor = collection.Find(Builders.Filter.Empty).ToCursor(); + var docList = await cursor.ToListAsync(); + rowFactory ??= getRowFactory(docList); + var firstTable = new BlueTableImmutable(rowFactory.Schema).ToDataPacketBuilder(); + foreach (var doc in docList) { + var row = rowFactory.GetBlueRow(doc); + firstTable.AddOrUpdate(row); + } + var dp = firstTable.ToDataPacket(); + lock (syncRoot) { + table = (BlueTableImmutable)dp.newRows; + subject.OnNext(dp); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbExtensions.cs b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbExtensions.cs new file mode 100644 index 000000000..a3106b088 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbExtensions.cs @@ -0,0 +1,63 @@ +namespace Bts.DataStream; + +using MongoDB.Bson; + +using System; + +public static class MongoDbExtensions { + public static BsonArray GetArray(this BsonDocument doc, string elementName) => + doc[elementName].AsBsonArray; + + public static BsonArray? GetArrayOrDefault(this BsonDocument doc, string elementName) => + doc.TryGetValue(elementName, out var val) && val.IsBsonArray ? + val.AsBsonArray : null; + + public static string GetString(this BsonDocument doc, string elementName) => + doc[elementName].AsString; + + public static string? GetStringOrDefault(this BsonDocument doc, string elementName) => + doc.TryGetValue(elementName, out var val) && val.IsString ? + val.AsString : null; + + public static int GetInt32(this BsonDocument doc, string elementName) => + doc[elementName].AsInt32; + + public static int? GetInt32OrDefault(this BsonDocument doc, string elementName) => + doc.TryGetValue(elementName, out var val) && val.IsInt32 ? + val.AsInt32 : null; + + public static long GetInt64(this BsonDocument doc, string elementName) => + doc[elementName].AsInt64; + + public static long? GetInt64OrDefault(this BsonDocument doc, string elementName) => + doc.TryGetValue(elementName, out var val) && val.IsInt64 ? + val.AsInt64 : null; + + public static double GetDouble(this BsonDocument doc, string elementName) => + doc[elementName].AsDouble; + + public static double? GetDoubleOrDefault(this BsonDocument doc, string elementName) => + doc.TryGetValue(elementName, out var val) && val.IsDouble ? + val.AsDouble : null; + + public static DateTime GetDateTime(this BsonDocument doc, string elementName) => + doc[elementName].ToUniversalTime(); + + public static DateTime? GetDateTimeOrDefault(this BsonDocument doc, string elementName) => + doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ? + val.ToUniversalTime() : null; + + public static DateTime GetTimestamp(this BsonDocument doc, string elementName) => + doc[elementName].ToUniversalTime(); + + public static DateTime? GetTimestampOrDefault(this BsonDocument doc, string elementName) => + doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ? + val.ToUniversalTime() : null; + + public static ObjectId GetObjectId(this BsonDocument doc, string elementName) => + doc[elementName].AsObjectId; + + public static ObjectId? GetObjectIdOrDefault(this BsonDocument doc, string elementName) => + doc.TryGetValue(elementName, out var val) && val.IsObjectId ? + val.AsObjectId : null; +} diff --git a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoSchemaAdaptor.cs b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoSchemaAdaptor.cs new file mode 100644 index 000000000..81389f068 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoSchemaAdaptor.cs @@ -0,0 +1,55 @@ +namespace Bts.DataStream; + +using bts; + +using MongoDB.Bson; + +using System; +using System.Collections.Generic; + +public static class MongoSchemaAdaptor { + public static BlueTableSchemaAdaptor InferSchema(IEnumerable docs) { + var schemaAdaptor = new BlueTableSchemaAdaptor("MongoCollectionStream"); + foreach (var doc in docs) { + foreach (var el in doc) { + if (schemaAdaptor.Contains(el.Name)) { + continue; + } + switch (el.Value.BsonType) { + case BsonType.Array: + schemaAdaptor.AddColumn(el.Name, d => d.GetArrayOrDefault(el.Name)?.ToString() ?? string.Empty); + break; + case BsonType.String: + schemaAdaptor.AddColumn(el.Name, d => d.GetStringOrDefault(el.Name) ?? string.Empty); + break; + case BsonType.Int32: + schemaAdaptor.AddColumn(el.Name, d => d.GetInt32OrDefault(el.Name)); + break; + case BsonType.Int64: + schemaAdaptor.AddColumn(el.Name, d => d.GetInt64OrDefault(el.Name)); + break; + case BsonType.Double: + schemaAdaptor.AddColumn(el.Name, d => d.GetDoubleOrDefault(el.Name)); + break; + case BsonType.DateTime: + schemaAdaptor.AddColumn(el.Name, d => d.GetDateTimeOrDefault(el.Name)); + break; + case BsonType.Timestamp: + schemaAdaptor.AddColumn(el.Name, d => d.GetTimestampOrDefault(el.Name)); + break; + case BsonType.ObjectId: + schemaAdaptor.AddColumn(el.Name, d => d.GetObjectIdOrDefault(el.Name)?.ToString()); + break; + case BsonType.Null: + Log.Gen.warn( + $"Ignoring column '{el.Name}' because at least one element is null"); + break; + default: + throw new InvalidOperationException( + $"Unsupported BsonType '{el.Value.BsonType}'"); + } + } + } + return schemaAdaptor; + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs new file mode 100644 index 000000000..a287faec8 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs @@ -0,0 +1,80 @@ +namespace Bts.DataStreamCore; + +using BCG; + +using System; +using System.Collections; +using System.Collections.Immutable; + +// I would have named this ImmutableBlueTable, but that was already taken. +public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection { + readonly ImmutableDictionary rows; + + public BlueTableImmutable(IBlueTableSchema schema) + : this(schema, ImmutableDictionary.Empty) { } + + public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary rows) { + this.schema = schema; + this.rows = rows; + } + + public int Count => rows.Count; + public bool IsSynchronized => false; + public IBlueTableSchema schema { get; } + public object SyncRoot => this; + + public IBlueRow this[int id] => rows[id]; + + public bool containsID(int id) => + rows.ContainsKey(id); + + public void CopyTo(Array array, int index) { + foreach (var r in rows.Values) { + array.SetValue(r, index++); + } + } + + public IEnumerator GetEnumerator() => + rows.Values.GetEnumerator(); + + public DataPacketBuilder ToDataPacketBuilder() => + new(this); + + public class DataPacketBuilder { + readonly IBlueTableSchema schema; + readonly ImmutableDictionary.Builder table; + readonly ImmutableDictionary.Builder added = + ImmutableDictionary.Empty.ToBuilder(); + readonly ImmutableDictionary.Builder removed = + ImmutableDictionary.Empty.ToBuilder(); + + public DataPacketBuilder(BlueTableImmutable table) { + schema = table.schema; + this.table = table.rows.ToBuilder(); + } + + public void AddOrUpdate(IBlueRow row) { + var id = row.Id(); + added[id] = row; + if (table.TryGetValue(id, out var orig)) { + removed.Remove(id); + } + } + + public void Remove(int id) { + added.Remove(id); + if (table.TryGetValue(id, out var orig)) { + removed[id] = orig; + } + } + + public DataPacket ToDataPacket() { + table.RemoveRange(removed.Keys); + table.AddRange(added); + var newRows = new BlueTableImmutable(schema, table.ToImmutable()); + var addedRows = new BlueTableImmutable(schema, added.ToImmutable()); + var removedRows = new BlueTableImmutable(schema, removed.ToImmutable()); + return new(newRows, addedRows, removedRows); + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdaptor.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdaptor.cs new file mode 100644 index 000000000..c7bd05612 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdaptor.cs @@ -0,0 +1,87 @@ +#nullable enable +namespace Bts.DataStream; + +using BCG; + +using bts.collections; + +using System; +using System.Collections.Generic; +using System.Linq; + +public class BlueTableSchemaAdaptor : IMutableBlueRowFactory { + readonly string name; + readonly List columns = new(); + BlueTableSchema? schema; + + public BlueTableSchemaAdaptor(string name) => + this.name = name; + + public IBlueTableSchema Schema => + schema ??= new BlueTableSchema(columns.WrapSelectorRO(c => c.Info).ToArray(), name); + + public bool Contains(string name) => + columns.Any(c => c.Info.colName == name); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r), typeof(int)); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(int)); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r), typeof(long)); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(long)); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r), typeof(double)); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(double)); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r), typeof(DateTime)); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(DateTime)); + + public void AddColumn(string name, Func getValue) => + AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(string)); + + public void AddColumn(string name, Func getValue, Type dataType) { + if (!ColumnInfo.isAllowedType(dataType)) { + throw new ArgumentException( + $"'{dataType}' is not a supported column type", nameof(dataType)); + } + // Invalidate cached schema (set it to null) + schema = null; + columns.Add(new ColumnAdaptor(new(name, columns.Count, dataType), r => getValue(r))); + } + + public IBlueRow GetBlueRow(T item) => + GetMutableBlueRow(item); + + public MutableBlueRow GetMutableBlueRow(T item) { + var mbr = new MutableBlueRow(Schema); + foreach (var col in columns) { + mbr.setValue(col.Info.colNo, col.GetValue(item)); + } + return mbr; + } + + class ColumnAdaptor { + readonly Func getValue; + + public ColumnAdaptor(ColumnInfo info, Func getValue) { + Info = info; + this.getValue = getValue; + } + + public ColumnInfo Info { get; } + + public object? GetValue(T row) => + getValue(row); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowFactory.cs b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowFactory.cs new file mode 100644 index 000000000..25ded906e --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowFactory.cs @@ -0,0 +1,9 @@ +#nullable enable +namespace Bts.DataStream; + +using BCG; + +public interface IBlueRowFactory { + IBlueTableSchema Schema { get; } + IBlueRow GetBlueRow(T item); +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/IMutableBlueRowFactory.cs b/src/DataStreamSuite/DataStreamCore/BlueData/IMutableBlueRowFactory.cs new file mode 100644 index 000000000..e1f182a92 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/IMutableBlueRowFactory.cs @@ -0,0 +1,11 @@ +#nullable enable +namespace Bts.DataStream; + +using BCG; + +public interface IMutableBlueRowFactory : IBlueRowFactory { + MutableBlueRow GetMutableBlueRow(T item); + + IBlueRow IBlueRowFactory.GetBlueRow(T item) => + GetMutableBlueRow(item); +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueRowExtensions.cs b/src/DataStreamSuite/DataStreamCore/BlueRowExtensions.cs new file mode 100644 index 000000000..6d36052e0 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueRowExtensions.cs @@ -0,0 +1,11 @@ +namespace Bts.DataStreamCore; + +using BCG; + +using System; + +public static class BlueRowExtensions { + public static int Id(this IBlueRow row) => + (int)(row["id"] ?? throw new ArgumentException( + "Row doesn't have an 'id' column", nameof(row))); +} diff --git a/src/DataStreamSuite/DataStreamCore/DataPacket.cs b/src/DataStreamSuite/DataStreamCore/DataPacket.cs index 441d2a06e..383f6073f 100644 --- a/src/DataStreamSuite/DataStreamCore/DataPacket.cs +++ b/src/DataStreamSuite/DataStreamCore/DataPacket.cs @@ -313,4 +313,9 @@ namespace BCG return String.Format("DataPacket, schema {0} n {1} d {2} a {3}", schema.tableName, newRows.Count, deletedRows.Count, addedRows.Count); } } + + public static class DataPacketExtensions { + public static bool HasChanges(this DataPacket dataPacket) => + dataPacket.addedRows.Count > 0 || dataPacket.deletedRows.Count > 0; + } }