feat: Add [incremental] MongoCollectionStream

This commit is contained in:
John Colvin committed 2024-05-25 15:09:42 -04:00
1 parent 250d9fb7cb
commit 97c82c4869
9 files changed
+524

No files matched your search

@@ -0,0 +1,203 @@
namespace Bts.DataStream;
using BCG;
using bts;
using Bts.DataStreamCore;
using mbts;
using MongoDB.Bson;
using MongoDB.Driver;
using Spark.Core.Disposables;
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Reactive.Subjects;
using System.Threading;
using System.Threading.Tasks;
public class MongoCollectionStream : ViewStream, IObservable<DataPacket> {
readonly string collectionName;
readonly string databaseName;
readonly string url;
readonly object syncRoot = new();
readonly CancellationTokenSource cts = new();
readonly Subject<DataPacket> subject = new();
bool hasStarted = false;
IBlueRowFactory<BsonDocument>? rowFactory;
MongoClient? clientBacking;
IMongoDatabase? databaseBacking;
IMongoCollection<BsonDocument>? collectionBacking = null;
BlueTableImmutable? table = null;
public MongoCollectionStream()
: this(
Environment.GetEnvironmentVariable("MONGO_DB_COLL") ?? string.Empty,
Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
}
public MongoCollectionStream(string collectionName)
: this(
collectionName,
Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
}
public MongoCollectionStream(string collectionName, string databaseName)
: this(
collectionName,
databaseName,
Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
}
public MongoCollectionStream(string collectionName, string databaseName, string url)
: this(collectionName, databaseName, url, null) { }
internal MongoCollectionStream(
string collectionName, string databaseName, string url, IBlueRowFactory<BsonDocument>? rowFactory) {
this.collectionName = collectionName;
this.databaseName = databaseName;
this.url = url;
this.rowFactory = rowFactory;
mySetRealSource(new ObservableViewStream(this));
}
MongoClient client => clientBacking ??= new(url);
IMongoDatabase database => databaseBacking ??= client.GetDatabase(databaseName);
IMongoCollection<BsonDocument>? collection {
get => Volatile.Read(ref collectionBacking);
set => Volatile.Write(ref collectionBacking, value);
}
public IDisposable Subscribe(IObserver<DataPacket> observer) {
lock (syncRoot) {
if (hasStarted) {
if (table is not null) {
// The first update has already been fired. Seed observer w/ current state.
observer.OnNext(new DataPacket(table, table, new BlueTable(table.schema)));
}
return subject.Subscribe(observer);
}
hasStarted = true;
}
var subscription = subject.Subscribe(observer);
// Retrieve collection snapshot to bootstrap sequence
initTable().ignore();
// Watch changes
Task.Run(watchChanges);
return Disposable.Create(Dispose);
void Dispose() {
Log.Gen.info("Disposing MongoDbChangeObservable subscription");
subscription.Dispose();
cts.Cancel();
cts.Dispose();
}
}
protected virtual IBlueRowFactory<BsonDocument> getRowFactory(IEnumerable<BsonDocument> docs) =>
MongoSchemaAdaptor.InferSchema(docs);
async Task watchChanges() {
Log.Gen.info(
$"MongoCollectionStreamSource. Watching changes. thread: {Environment.CurrentManagedThreadId}");
while (collection is null) {
await Task.Delay(100);
}
Log.notnull(collection);
using var cursor = collection.Watch(new ChangeStreamOptions {
FullDocument = ChangeStreamFullDocumentOption.UpdateLookup
});
try {
while (await cursor.MoveNextAsync(cts.Token)) {
// Ignore while waiting for init
lock (syncRoot) {
if (table is null) {
continue;
}
}
// schemaAdaptor is initialized after table; redundantly assert both
// are non-null to be safe, and clarify expectations.
Log.notnull(rowFactory);
Log.notnull(table);
var bd = table.ToDataPacketBuilder();
foreach (var ch in cursor.Current) {
var opDescJson = ch.OperationDescription?.ToJson() ?? string.Empty;
var updDescJson = ch.UpdateDescription?.ToJson() ?? string.Empty;
Log.Gen.info(
$"MongoCollectionStream.watchChanges. Doc changed. type: {ch.OperationType} " +
$"changes:\n\t\t{updDescJson}");
switch (ch.OperationType) {
case ChangeStreamOperationType.Insert: {
bd.AddOrUpdate(Row());
break;
}
case ChangeStreamOperationType.Update: {
bd.Remove(OldRow().Id());
bd.AddOrUpdate(Row());
break;
}
case ChangeStreamOperationType.Replace: {
bd.Remove(OldRow().Id());
bd.AddOrUpdate(Row());
break;
}
case ChangeStreamOperationType.Delete: {
bd.Remove(OldRow().Id());
break;
}
default:
Log.assert(false, $"Unsupported op type '{ch.OperationType}'");
break;
}
IBlueRow Row() =>
rowFactory.GetBlueRow(ch.FullDocumentBeforeChange);
IBlueRow OldRow() =>
rowFactory.GetBlueRow(ch.FullDocumentBeforeChange);
}
var dp = bd.ToDataPacket();
if (!dp.HasChanges()) {
Log.Gen.warn(
$"MongoDbChangeObservable Ignoring update because it didn't change any rows.");
continue;
}
lock (syncRoot) {
table = (BlueTableImmutable)dp.newRows;
subject.OnNext(dp);
}
}
} catch (TaskCanceledException ex) {
Log.Gen.warn(
$"MongoDbChangeObservable 'Watch' task cancelled (this is " +
$"expected if subscription was disposed). {ex}");
}
}
[MemberNotNull(nameof(collection))]
async Task initTable() {
collection = database.GetCollection<BsonDocument>(collectionName);
var cursor = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
var docList = await cursor.ToListAsync();
rowFactory ??= getRowFactory(docList);
var firstTable = new BlueTableImmutable(rowFactory.Schema).ToDataPacketBuilder();
foreach (var doc in docList) {
var row = rowFactory.GetBlueRow(doc);
firstTable.AddOrUpdate(row);
}
var dp = firstTable.ToDataPacket();
lock (syncRoot) {
table = (BlueTableImmutable)dp.newRows;
subject.OnNext(dp);
}
}
}
@@ -0,0 +1,63 @@
namespace Bts.DataStream;
using MongoDB.Bson;
using System;
public static class MongoDbExtensions {
public static BsonArray GetArray(this BsonDocument doc, string elementName) =>
doc[elementName].AsBsonArray;
public static BsonArray? GetArrayOrDefault(this BsonDocument doc, string elementName) =>
doc.TryGetValue(elementName, out var val) && val.IsBsonArray ?
val.AsBsonArray : null;
public static string GetString(this BsonDocument doc, string elementName) =>
doc[elementName].AsString;
public static string? GetStringOrDefault(this BsonDocument doc, string elementName) =>
doc.TryGetValue(elementName, out var val) && val.IsString ?
val.AsString : null;
public static int GetInt32(this BsonDocument doc, string elementName) =>
doc[elementName].AsInt32;
public static int? GetInt32OrDefault(this BsonDocument doc, string elementName) =>
doc.TryGetValue(elementName, out var val) && val.IsInt32 ?
val.AsInt32 : null;
public static long GetInt64(this BsonDocument doc, string elementName) =>
doc[elementName].AsInt64;
public static long? GetInt64OrDefault(this BsonDocument doc, string elementName) =>
doc.TryGetValue(elementName, out var val) && val.IsInt64 ?
val.AsInt64 : null;
public static double GetDouble(this BsonDocument doc, string elementName) =>
doc[elementName].AsDouble;
public static double? GetDoubleOrDefault(this BsonDocument doc, string elementName) =>
doc.TryGetValue(elementName, out var val) && val.IsDouble ?
val.AsDouble : null;
public static DateTime GetDateTime(this BsonDocument doc, string elementName) =>
doc[elementName].ToUniversalTime();
public static DateTime? GetDateTimeOrDefault(this BsonDocument doc, string elementName) =>
doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
val.ToUniversalTime() : null;
public static DateTime GetTimestamp(this BsonDocument doc, string elementName) =>
doc[elementName].ToUniversalTime();
public static DateTime? GetTimestampOrDefault(this BsonDocument doc, string elementName) =>
doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
val.ToUniversalTime() : null;
public static ObjectId GetObjectId(this BsonDocument doc, string elementName) =>
doc[elementName].AsObjectId;
public static ObjectId? GetObjectIdOrDefault(this BsonDocument doc, string elementName) =>
doc.TryGetValue(elementName, out var val) && val.IsObjectId ?
val.AsObjectId : null;
}
@@ -0,0 +1,55 @@
namespace Bts.DataStream;
using bts;
using MongoDB.Bson;
using System;
using System.Collections.Generic;
public static class MongoSchemaAdaptor {
public static BlueTableSchemaAdaptor<BsonDocument> InferSchema(IEnumerable<BsonDocument> docs) {
var schemaAdaptor = new BlueTableSchemaAdaptor<BsonDocument>("MongoCollectionStream");
foreach (var doc in docs) {
foreach (var el in doc) {
if (schemaAdaptor.Contains(el.Name)) {
continue;
}
switch (el.Value.BsonType) {
case BsonType.Array:
schemaAdaptor.AddColumn(el.Name, d => d.GetArrayOrDefault(el.Name)?.ToString() ?? string.Empty);
break;
case BsonType.String:
schemaAdaptor.AddColumn(el.Name, d => d.GetStringOrDefault(el.Name) ?? string.Empty);
break;
case BsonType.Int32:
schemaAdaptor.AddColumn(el.Name, d => d.GetInt32OrDefault(el.Name));
break;
case BsonType.Int64:
schemaAdaptor.AddColumn(el.Name, d => d.GetInt64OrDefault(el.Name));
break;
case BsonType.Double:
schemaAdaptor.AddColumn(el.Name, d => d.GetDoubleOrDefault(el.Name));
break;
case BsonType.DateTime:
schemaAdaptor.AddColumn(el.Name, d => d.GetDateTimeOrDefault(el.Name));
break;
case BsonType.Timestamp:
schemaAdaptor.AddColumn(el.Name, d => d.GetTimestampOrDefault(el.Name));
break;
case BsonType.ObjectId:
schemaAdaptor.AddColumn(el.Name, d => d.GetObjectIdOrDefault(el.Name)?.ToString());
break;
case BsonType.Null:
Log.Gen.warn(
$"Ignoring column '{el.Name}' because at least one element is null");
break;
default:
throw new InvalidOperationException(
$"Unsupported BsonType '{el.Value.BsonType}'");
}
}
}
return schemaAdaptor;
}
}
@@ -0,0 +1,80 @@
namespace Bts.DataStreamCore;
using BCG;
using System;
using System.Collections;
using System.Collections.Immutable;
// I would have named this ImmutableBlueTable, but that was already taken.
public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
readonly ImmutableDictionary<int, IBlueRow> rows;
public BlueTableImmutable(IBlueTableSchema schema)
: this(schema, ImmutableDictionary<int, IBlueRow>.Empty) { }
public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary<int, IBlueRow> rows) {
this.schema = schema;
this.rows = rows;
}
public int Count => rows.Count;
public bool IsSynchronized => false;
public IBlueTableSchema schema { get; }
public object SyncRoot => this;
public IBlueRow this[int id] => rows[id];
public bool containsID(int id) =>
rows.ContainsKey(id);
public void CopyTo(Array array, int index) {
foreach (var r in rows.Values) {
array.SetValue(r, index++);
}
}
public IEnumerator GetEnumerator() =>
rows.Values.GetEnumerator();
public DataPacketBuilder ToDataPacketBuilder() =>
new(this);
public class DataPacketBuilder {
readonly IBlueTableSchema schema;
readonly ImmutableDictionary<int, IBlueRow>.Builder table;
readonly ImmutableDictionary<int, IBlueRow>.Builder added =
ImmutableDictionary<int, IBlueRow>.Empty.ToBuilder();
readonly ImmutableDictionary<int, IBlueRow>.Builder removed =
ImmutableDictionary<int, IBlueRow>.Empty.ToBuilder();
public DataPacketBuilder(BlueTableImmutable table) {
schema = table.schema;
this.table = table.rows.ToBuilder();
}
public void AddOrUpdate(IBlueRow row) {
var id = row.Id();
added[id] = row;
if (table.TryGetValue(id, out var orig)) {
removed.Remove(id);
}
}
public void Remove(int id) {
added.Remove(id);
if (table.TryGetValue(id, out var orig)) {
removed[id] = orig;
}
}
public DataPacket ToDataPacket() {
table.RemoveRange(removed.Keys);
table.AddRange(added);
var newRows = new BlueTableImmutable(schema, table.ToImmutable());
var addedRows = new BlueTableImmutable(schema, added.ToImmutable());
var removedRows = new BlueTableImmutable(schema, removed.ToImmutable());
return new(newRows, addedRows, removedRows);
}
}
}
@@ -0,0 +1,87 @@
#nullable enable
namespace Bts.DataStream;
using BCG;
using bts.collections;
using System;
using System.Collections.Generic;
using System.Linq;
public class BlueTableSchemaAdaptor<T> : IMutableBlueRowFactory<T> {
readonly string name;
readonly List<ColumnAdaptor> columns = new();
BlueTableSchema? schema;
public BlueTableSchemaAdaptor(string name) =>
this.name = name;
public IBlueTableSchema Schema =>
schema ??= new BlueTableSchema(columns.WrapSelectorRO(c => c.Info).ToArray(), name);
public bool Contains(string name) =>
columns.Any(c => c.Info.colName == name);
public void AddColumn(string name, Func<T, int> getValue) =>
AddColumn(name, r => getValue(r), typeof(int));
public void AddColumn(string name, Func<T, int?> getValue) =>
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(int));
public void AddColumn(string name, Func<T, long> getValue) =>
AddColumn(name, r => getValue(r), typeof(long));
public void AddColumn(string name, Func<T, long?> getValue) =>
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(long));
public void AddColumn(string name, Func<T, double> getValue) =>
AddColumn(name, r => getValue(r), typeof(double));
public void AddColumn(string name, Func<T, double?> getValue) =>
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(double));
public void AddColumn(string name, Func<T, DateTime> getValue) =>
AddColumn(name, r => getValue(r), typeof(DateTime));
public void AddColumn(string name, Func<T, DateTime?> getValue) =>
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(DateTime));
public void AddColumn(string name, Func<T, string?> getValue) =>
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(string));
public void AddColumn(string name, Func<T, object?> getValue, Type dataType) {
if (!ColumnInfo.isAllowedType(dataType)) {
throw new ArgumentException(
$"'{dataType}' is not a supported column type", nameof(dataType));
}
// Invalidate cached schema (set it to null)
schema = null;
columns.Add(new ColumnAdaptor(new(name, columns.Count, dataType), r => getValue(r)));
}
public IBlueRow GetBlueRow(T item) =>
GetMutableBlueRow(item);
public MutableBlueRow GetMutableBlueRow(T item) {
var mbr = new MutableBlueRow(Schema);
foreach (var col in columns) {
mbr.setValue(col.Info.colNo, col.GetValue(item));
}
return mbr;
}
class ColumnAdaptor {
readonly Func<T, object?> getValue;
public ColumnAdaptor(ColumnInfo info, Func<T, object?> getValue) {
Info = info;
this.getValue = getValue;
}
public ColumnInfo Info { get; }
public object? GetValue(T row) =>
getValue(row);
}
}
@@ -0,0 +1,9 @@
#nullable enable
namespace Bts.DataStream;
using BCG;
public interface IBlueRowFactory<T> {
IBlueTableSchema Schema { get; }
IBlueRow GetBlueRow(T item);
}
@@ -0,0 +1,11 @@
#nullable enable
namespace Bts.DataStream;
using BCG;
public interface IMutableBlueRowFactory<T> : IBlueRowFactory<T> {
MutableBlueRow GetMutableBlueRow(T item);
IBlueRow IBlueRowFactory<T>.GetBlueRow(T item) =>
GetMutableBlueRow(item);
}
@@ -0,0 +1,11 @@
namespace Bts.DataStreamCore;
using BCG;
using System;
public static class BlueRowExtensions {
public static int Id(this IBlueRow row) =>
(int)(row["id"] ?? throw new ArgumentException(
"Row doesn't have an 'id' column", nameof(row)));
}
@@ -313,4 +313,9 @@ namespace BCG
return String.Format("DataPacket, schema {0} n {1} d {2} a {3}", schema.tableName, newRows.Count, deletedRows.Count, addedRows.Count);
}
}
public static class DataPacketExtensions {
public static bool HasChanges(this DataPacket dataPacket) =>
dataPacket.addedRows.Count > 0 || dataPacket.deletedRows.Count > 0;
}
}