feat: Add IncrementalMongoDbStream
This commit is contained in:
1 parent
56938c780b
commit
15f811ebfb
7 files changed
+463
-1
No files matched your search
@@ -6,13 +6,18 @@ using bts;
|
||||
|
||||
using System;
|
||||
|
||||
class ObservableViewStream : ViewStream, IObservable<DataPacket> {
|
||||
public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
|
||||
readonly IObservable<DataPacket> source;
|
||||
IDisposable? subscription;
|
||||
|
||||
public ObservableViewStream(IObservable<DataPacket> source) =>
|
||||
this.source = source;
|
||||
|
||||
public void Dispose() {
|
||||
subscription?.Dispose();
|
||||
subscription = null;
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
Log.isnull(subscription);
|
||||
subscription = source.Subscribe(myFireEvent);
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
namespace Bts.DataStream;
|
||||
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
public sealed class IncrementalMongoDbStream : ViewStream {
|
||||
readonly string collectionName;
|
||||
readonly string databaseName;
|
||||
readonly string url;
|
||||
readonly string index;
|
||||
readonly Dictionary<string, int> idMap = [];
|
||||
readonly Dictionary<string, string> columnMap = [];
|
||||
readonly MongoDbChangeStreamObservable source;
|
||||
|
||||
public IncrementalMongoDbStream()
|
||||
: this(
|
||||
Environment.GetEnvironmentVariable("MONGO_DB_COLL") ?? string.Empty,
|
||||
Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
|
||||
Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
|
||||
}
|
||||
|
||||
public IncrementalMongoDbStream(string collectionName, string databaseName, string url)
|
||||
: this(collectionName, databaseName, url, "_id") { }
|
||||
|
||||
public IncrementalMongoDbStream(string collectionName, string databaseName, string url, string index) {
|
||||
this.collectionName = collectionName;
|
||||
this.databaseName = databaseName;
|
||||
this.url = url;
|
||||
this.index = index;
|
||||
source = new(collectionName, databaseName, url);
|
||||
mySetRealSource(source.AsViewStream());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
namespace Bts.DataStream;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts;
|
||||
using bts.collections;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using mbts;
|
||||
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
using Spark.Core;
|
||||
using Spark.Core.Disposables;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Reactive.Subjects;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
public class MongoDbChangeStreamObservable : IObservable<DataPacket> {
|
||||
readonly string collectionName;
|
||||
readonly string databaseName;
|
||||
readonly string url;
|
||||
readonly object locker = new();
|
||||
readonly CancellationTokenSource cts = new();
|
||||
readonly BlueTableSchemaAdaptor<BsonDocument> schemaBd = new("MongoDbChangeStream");
|
||||
|
||||
bool hasStarted = false;
|
||||
MongoClient? clientBacking;
|
||||
IMongoDatabase? databaseBacking;
|
||||
IMongoCollection<BsonDocument>? collectionBacking = null;
|
||||
BlueTableImmutable? table = null;
|
||||
Subject<DataPacket> subject = new();
|
||||
|
||||
public MongoDbChangeStreamObservable(string collectionName, string databaseName, string url) {
|
||||
this.collectionName = collectionName;
|
||||
this.databaseName = databaseName;
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
MongoClient client => clientBacking ??= new(url);
|
||||
IMongoDatabase database => databaseBacking ??= client.GetDatabase(databaseName);
|
||||
IMongoCollection<BsonDocument>? collection {
|
||||
get => Volatile.Read(ref collectionBacking);
|
||||
set => Volatile.Write(ref collectionBacking, value);
|
||||
}
|
||||
|
||||
public IDisposable Subscribe(IObserver<DataPacket> observer) {
|
||||
lock (locker) {
|
||||
if (hasStarted) {
|
||||
if (table is not null) {
|
||||
// The first update has already been fired. Seed observer w/ current state.
|
||||
observer.OnNext(new DataPacket(table, table, new BlueTable(table.schema)));
|
||||
}
|
||||
return subject.Subscribe(observer);
|
||||
}
|
||||
hasStarted = true;
|
||||
}
|
||||
var subscription = subject.Subscribe(observer);
|
||||
// Retrieve collection snapshot in background
|
||||
initTable().ignore();
|
||||
// Listen for changes in background
|
||||
Listen().ignore();
|
||||
return Disposable.Create(Dispose);
|
||||
|
||||
void Dispose() {
|
||||
Log.Gen.info("Disposing MongoDbChangeObservable subscription");
|
||||
subscription.Dispose();
|
||||
cts.Cancel();
|
||||
cts.Dispose();
|
||||
}
|
||||
|
||||
async Task Listen() {
|
||||
while (collection is null) {
|
||||
Thread.Yield();
|
||||
}
|
||||
using var cursor = collection.Watch(new ChangeStreamOptions {
|
||||
FullDocument = ChangeStreamFullDocumentOption.UpdateLookup
|
||||
});
|
||||
try {
|
||||
while (await cursor.MoveNextAsync(cts.Token)) {
|
||||
// Ignore changes while waiting for collection snapshot
|
||||
lock (locker) {
|
||||
if (table is null) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
var bd = table.ToDataPacketBuilder();
|
||||
foreach (var ch in cursor.Current) {
|
||||
var opDescJson = ch.OperationDescription?.ToJson() ?? string.Empty;
|
||||
var updDescJson = ch.UpdateDescription?.ToJson() ?? string.Empty;
|
||||
var doc = ch.FullDocument;
|
||||
var docJson = doc.ToJson();
|
||||
Log.Gen.info(
|
||||
$"IncrementalMongoDbStream doc changed. type: {ch.OperationType} " +
|
||||
$"\n\tchanges:\n\t\t{updDescJson}\n\tnewValue:\n\t\t{docJson}");
|
||||
switch (ch.OperationType) {
|
||||
case ChangeStreamOperationType.Insert: {
|
||||
Log.Gen.info(
|
||||
$"IncrementalMongoDbStream doc changed. type: {ch.OperationType} " +
|
||||
$"\n\tchanges:\n\t\t{updDescJson}\n\tnewValue:\n\t\t{docJson}");
|
||||
var row = schemaBd.GetBlueRow(doc);
|
||||
bd.AddOrUpdate(row);
|
||||
break;
|
||||
}
|
||||
case ChangeStreamOperationType.Update: {
|
||||
var oldRow = schemaBd.GetBlueRow(ch.FullDocumentBeforeChange);
|
||||
bd.Remove(oldRow.Id());
|
||||
var newRow = schemaBd.GetBlueRow(ch.FullDocument);
|
||||
bd.AddOrUpdate(newRow);
|
||||
break;
|
||||
}
|
||||
case ChangeStreamOperationType.Replace: {
|
||||
var oldRow = schemaBd.GetBlueRow(ch.FullDocumentBeforeChange);
|
||||
bd.Remove(oldRow.Id());
|
||||
var newRow = schemaBd.GetBlueRow(ch.FullDocument);
|
||||
bd.AddOrUpdate(newRow);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
Log.assert(false, $"Unsupported op type '{ch.OperationType}'");
|
||||
break;
|
||||
}
|
||||
}
|
||||
var dp = bd.ToDataPacket();
|
||||
if (!dp.HasChanges()) {
|
||||
Log.Gen.warn(
|
||||
$"MongoDbChangeObservable Ignoring update because it didn't change any rows.");
|
||||
continue;
|
||||
}
|
||||
lock (locker) {
|
||||
table = (BlueTableImmutable)dp.newRows;
|
||||
subject.OnNext(dp);
|
||||
}
|
||||
}
|
||||
} catch (TaskCanceledException ex) {
|
||||
Log.Gen.warn(
|
||||
$"MongoDbChangeObservable 'Watch' task cancelled (this is " +
|
||||
$"expected if subscription was disposed). {ex}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async Task initTable() {
|
||||
collection = database.GetCollection<BsonDocument>(collectionName);
|
||||
var cursor = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
||||
var docList = await cursor.ToListAsync();
|
||||
initSchema(docList);
|
||||
var firstTable = new BlueTableImmutable(schemaBd.Schema).ToDataPacketBuilder();
|
||||
foreach (var doc in docList) {
|
||||
var row = schemaBd.GetBlueRow(doc);
|
||||
firstTable.AddOrUpdate(row);
|
||||
}
|
||||
var dp = firstTable.ToDataPacket();
|
||||
lock (locker) {
|
||||
table = (BlueTableImmutable)dp.newRows;
|
||||
subject.OnNext(dp);
|
||||
}
|
||||
}
|
||||
|
||||
void initSchema(IEnumerable<BsonDocument> docs) {
|
||||
foreach (var doc in docs) {
|
||||
foreach (var el in doc) {
|
||||
if (schemaBd.Contains(el.Name)) {
|
||||
continue;
|
||||
}
|
||||
switch (el.Value.BsonType) {
|
||||
case BsonType.Array:
|
||||
schemaBd.AddColumn(el.Name, d => d.GetArrayOrDefault(el.Name)?.ToString() ?? string.Empty);
|
||||
break;
|
||||
case BsonType.String:
|
||||
schemaBd.AddColumn(el.Name, d => d.GetStringOrDefault(el.Name) ?? string.Empty);
|
||||
break;
|
||||
case BsonType.Int32:
|
||||
schemaBd.AddColumn(el.Name, d => d.GetInt32OrDefault(el.Name));
|
||||
break;
|
||||
case BsonType.Int64:
|
||||
schemaBd.AddColumn(el.Name, d => d.GetInt64OrDefault(el.Name));
|
||||
break;
|
||||
case BsonType.Double:
|
||||
schemaBd.AddColumn(el.Name, d => d.GetDoubleOrDefault(el.Name));
|
||||
break;
|
||||
case BsonType.DateTime:
|
||||
schemaBd.AddColumn(el.Name, d => d.GetDateTimeOrDefault(el.Name));
|
||||
break;
|
||||
case BsonType.Timestamp:
|
||||
schemaBd.AddColumn(el.Name, d => d.GetTimestampOrDefault(el.Name));
|
||||
break;
|
||||
case BsonType.ObjectId:
|
||||
schemaBd.AddColumn(el.Name, d => d.GetObjectIdOrDefault(el.Name)?.ToString());
|
||||
break;
|
||||
case BsonType.Null:
|
||||
Log.Gen.warn(
|
||||
$"Ignoring column '{el.Name}' because at least one element is null");
|
||||
break;
|
||||
default:
|
||||
throw new InvalidOperationException(
|
||||
$"Unsupported BsonType '{el.Value.BsonType}'");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
namespace Bts.DataStream;
|
||||
|
||||
using MongoDB.Bson;
|
||||
|
||||
using System;
|
||||
|
||||
public static class MongoDbExtensions {
|
||||
public static BsonArray? GetArrayOrDefault(this BsonDocument doc, string elementName) =>
|
||||
doc.TryGetValue(elementName, out var val) && val.IsBsonArray ?
|
||||
val.AsBsonArray : null;
|
||||
|
||||
public static string? GetStringOrDefault(this BsonDocument doc, string elementName) =>
|
||||
doc.TryGetValue(elementName, out var val) && val.IsString ?
|
||||
val.AsString : null;
|
||||
|
||||
public static int? GetInt32OrDefault(this BsonDocument doc, string elementName) =>
|
||||
doc.TryGetValue(elementName, out var val) && val.IsInt32 ?
|
||||
val.AsInt32 : null;
|
||||
|
||||
public static long? GetInt64OrDefault(this BsonDocument doc, string elementName) =>
|
||||
doc.TryGetValue(elementName, out var val) && val.IsInt64 ?
|
||||
val.AsInt64 : null;
|
||||
|
||||
public static double? GetDoubleOrDefault(this BsonDocument doc, string elementName) =>
|
||||
doc.TryGetValue(elementName, out var val) && val.IsDouble ?
|
||||
val.AsDouble : null;
|
||||
|
||||
public static DateTime? GetDateTimeOrDefault(this BsonDocument doc, string elementName) =>
|
||||
doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
|
||||
val.ToUniversalTime() : null;
|
||||
|
||||
public static DateTime? GetTimestampOrDefault(this BsonDocument doc, string elementName) =>
|
||||
doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
|
||||
val.ToUniversalTime() : null;
|
||||
|
||||
public static ObjectId? GetObjectIdOrDefault(this BsonDocument doc, string elementName) =>
|
||||
doc.TryGetValue(elementName, out var val) && val.IsObjectId ?
|
||||
val.AsObjectId : null;
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
namespace Bts.DataStreamCore;
|
||||
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Immutable;
|
||||
|
||||
// I would have named this ImmutableBlueTable, but that was already taken.
|
||||
public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
|
||||
readonly ImmutableDictionary<int, IBlueRow> rows;
|
||||
|
||||
public BlueTableImmutable(IBlueTableSchema schema)
|
||||
: this(schema, ImmutableDictionary<int, IBlueRow>.Empty) { }
|
||||
|
||||
public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary<int, IBlueRow> rows) {
|
||||
this.schema = schema;
|
||||
this.rows = rows;
|
||||
}
|
||||
|
||||
public int Count => rows.Count;
|
||||
public bool IsSynchronized => false;
|
||||
public IBlueTableSchema schema { get; }
|
||||
public object SyncRoot => this;
|
||||
|
||||
public IBlueRow this[int id] => rows[id];
|
||||
|
||||
public bool containsID(int id) =>
|
||||
rows.ContainsKey(id);
|
||||
|
||||
public void CopyTo(Array array, int index) {
|
||||
foreach (var r in rows.Values) {
|
||||
array.SetValue(r, index++);
|
||||
}
|
||||
}
|
||||
|
||||
public IEnumerator GetEnumerator() =>
|
||||
rows.Values.GetEnumerator();
|
||||
|
||||
public DataPacketBuilder ToDataPacketBuilder() =>
|
||||
new(this);
|
||||
|
||||
public class DataPacketBuilder {
|
||||
readonly IBlueTableSchema schema;
|
||||
readonly ImmutableDictionary<int, IBlueRow>.Builder table;
|
||||
readonly ImmutableDictionary<int, IBlueRow>.Builder added =
|
||||
ImmutableDictionary<int, IBlueRow>.Empty.ToBuilder();
|
||||
readonly ImmutableDictionary<int, IBlueRow>.Builder removed =
|
||||
ImmutableDictionary<int, IBlueRow>.Empty.ToBuilder();
|
||||
|
||||
public DataPacketBuilder(BlueTableImmutable table) {
|
||||
schema = table.schema;
|
||||
this.table = table.rows.ToBuilder();
|
||||
}
|
||||
|
||||
public void AddOrUpdate(IBlueRow row) {
|
||||
var id = row.Id();
|
||||
added[id] = row;
|
||||
if (table.TryGetValue(id, out var orig)) {
|
||||
removed.Remove(id);
|
||||
}
|
||||
}
|
||||
|
||||
public void Remove(int id) {
|
||||
added.Remove(id);
|
||||
if (table.TryGetValue(id, out var orig)) {
|
||||
removed[id] = orig;
|
||||
}
|
||||
}
|
||||
|
||||
public DataPacket ToDataPacket() {
|
||||
table.RemoveRange(removed.Keys);
|
||||
table.AddRange(added);
|
||||
var newRows = new BlueTableImmutable(schema, table.ToImmutable());
|
||||
var addedRows = new BlueTableImmutable(schema, added.ToImmutable());
|
||||
var removedRows = new BlueTableImmutable(schema, removed.ToImmutable());
|
||||
return new(newRows, addedRows, removedRows);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
#nullable enable
|
||||
namespace Bts.DataStream;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts.collections;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
public class BlueTableSchemaAdaptor<T> {
|
||||
readonly string name;
|
||||
readonly List<ColumnAdaptor> columns = [];
|
||||
BlueTableSchema? schema;
|
||||
|
||||
public BlueTableSchemaAdaptor(string name) =>
|
||||
this.name = name;
|
||||
|
||||
public IBlueTableSchema Schema =>
|
||||
schema ??= new BlueTableSchema([.. columns.WrapSelectorRO(c => c.Info)], name);
|
||||
|
||||
public bool Contains(string name) =>
|
||||
columns.Any(c => c.Info.colName == name);
|
||||
|
||||
public void AddColumn(string name, Func<T, int> getValue) =>
|
||||
AddColumn(name, r => getValue(r), typeof(int));
|
||||
|
||||
public void AddColumn(string name, Func<T, int?> getValue) =>
|
||||
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(int));
|
||||
|
||||
public void AddColumn(string name, Func<T, long> getValue) =>
|
||||
AddColumn(name, r => getValue(r), typeof(long));
|
||||
|
||||
public void AddColumn(string name, Func<T, long?> getValue) =>
|
||||
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(long));
|
||||
|
||||
public void AddColumn(string name, Func<T, double> getValue) =>
|
||||
AddColumn(name, r => getValue(r), typeof(double));
|
||||
|
||||
public void AddColumn(string name, Func<T, double?> getValue) =>
|
||||
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(double));
|
||||
|
||||
public void AddColumn(string name, Func<T, DateTime> getValue) =>
|
||||
AddColumn(name, r => getValue(r), typeof(DateTime));
|
||||
|
||||
public void AddColumn(string name, Func<T, DateTime?> getValue) =>
|
||||
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(DateTime));
|
||||
|
||||
public void AddColumn(string name, Func<T, string?> getValue) =>
|
||||
AddColumn(name, r => getValue(r) ?? (object)DBNull.Value, typeof(string));
|
||||
|
||||
public void AddColumn(string name, Func<T, object?> getValue, Type dataType) {
|
||||
if (!ColumnInfo.isAllowedType(dataType)) {
|
||||
throw new ArgumentException(
|
||||
$"'{dataType}' is not a supported column type", nameof(dataType));
|
||||
}
|
||||
// Invalidate cached schema (set it to null)
|
||||
schema = null;
|
||||
columns.Add(new ColumnAdaptor(new(name, columns.Count, dataType), r => getValue(r)));
|
||||
}
|
||||
|
||||
public IBlueRow GetBlueRow(T item) {
|
||||
var mbr = new MutableBlueRow(Schema);
|
||||
foreach (var col in columns) {
|
||||
mbr.setValue(col.Info.colNo, col.GetValue(item));
|
||||
}
|
||||
return mbr;
|
||||
}
|
||||
|
||||
class ColumnAdaptor {
|
||||
readonly Func<T, object?> getValue;
|
||||
|
||||
public ColumnAdaptor(ColumnInfo info, Func<T, object?> getValue) {
|
||||
Info = info;
|
||||
this.getValue = getValue;
|
||||
}
|
||||
|
||||
public ColumnInfo Info { get; }
|
||||
|
||||
public object? GetValue(T row) =>
|
||||
getValue(row);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace Bts.DataStreamCore;
|
||||
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
|
||||
public static class BlueRowExtensions {
|
||||
public static int Id(this IBlueRow row) =>
|
||||
(int)(row["id"] ?? throw new ArgumentException(
|
||||
"Row doesn't have an 'id' column", nameof(row)));
|
||||
}
|
||||
Reference in new issue
Block a user