diff --git a/src/DataStreamSuite/DataStream/BlueOrb/ApplyStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/ApplyStream.cs new file mode 100644 index 000000000..8cb3343a5 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/ApplyStream.cs @@ -0,0 +1,93 @@ +#nullable enable +namespace bts.BlueOrb; + +using BCG; + +using Bts.DataStream; +using Bts.DataStreamCore; + +using Spark.Core.Disposables; + +using System; + +public class ApplyStream : ViewStream { + readonly ViewStream source; + readonly Func apply; + IBlueTableSchema? outputSchema; + BlueTableImmutable? lastFired; + IDisposable? sub; + + public ApplyStream(ViewStream source, IFunctionEvaluator applyEvaluator) { + this.source = source; + apply = Apply; + + object? Apply(IBlueRow r) { + var res = applyEvaluator.Evaluate(string.Empty, new (string?, int?, object?)[] { (null, 0, r) }); + return res; + } + } + + public ApplyStream(ViewStream source, Func apply) { + this.source = source; + this.apply = apply; + } + + protected override void myOnFirstSubscriber() { + sub = source.Subscribe(OnSourceChanged); + + void OnSourceChanged(IDataPacket dp) { + Lazy lastFired = new(() => this.lastFired ??= new(Log.notnull(outputSchema))); + Lazy bd = new(() => lastFired.Value.ToDataPacketBuilder()); + + foreach (var r in dp.deletedRows) { + var res = apply(r); + if (res is IBlueRow blueRow) { + bd.Value.Remove(blueRow.Id()); + } else if (res is IBlueRowCollection blueRows) { + foreach (var br in blueRows) { + bd.Value.Remove(br.Id()); + } + } else { + Log.assert(false, $"apply delegate returned obj of unexpected type '{res?.GetType()}'"); + } + } + foreach (var r in dp.addedRows) { + var res = apply(r); + if (res is IBlueRow blueRow) { + outputSchema ??= blueRow.schema; + bd.Value.AddOrUpdate(blueRow); + } else if (res is IBlueRowCollection blueRows) { + outputSchema ??= blueRows.schema; + foreach (var br in blueRows) { + bd.Value.AddOrUpdate(br); + } + } else { + if (outputSchema is null) { + // We get the outputSchema from the result of the first apply, if it's null, we + // cannot determine the output data type. + // TODO: maybe support specifying return type in DSL? + Log.notnull(res); + } + var resType = res?.GetType(); + if (outputSchema is null) { + Log.notnull(resType); + Log.assert(ColumnInfo.isAllowedType(resType)); + outputSchema = new BlueTableSchema(new ColumnInfo[] { + new("id", 0, typeof(int)), + new("result", 1, resType), + }, dp.newRows.schema.tableName + "_Apply"); + } + bd.Value.AddOrUpdate(new BlueRow(outputSchema, new object?[] { r.Id(), res })); + // Log.assert(false, $"apply delegate returned obj of unexpected type '{res?.GetType()}'"); + } + } + if (bd.IsValueCreated) { + myFireEvent(bd.Value.ToDataPacket(out this.lastFired)); + } + } + } + + protected override void myOnLastSubscriber() { + Disposable.dispose(ref sub); + } +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/IndexRow.cs b/src/DataStreamSuite/DataStream/BlueOrb/IndexRow.cs new file mode 100644 index 000000000..772a66c48 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/IndexRow.cs @@ -0,0 +1,23 @@ +namespace bts.BlueOrb; + +using BCG; + +public class IndexRow : IBlueRow { + readonly IBlueRow inputRow; + readonly int id; + readonly int idIndex; + + public IndexRow(IBlueRow inputRow, int id, int idIndex, IBlueTableSchema schema) { + this.inputRow = inputRow; + this.id = id; + this.idIndex = idIndex; + this.schema = schema; + } + + public IBlueRow InputRow => inputRow; + + public IBlueTableSchema schema { get; } + + public object? this[int index] => + index == idIndex ? id : inputRow[index]; +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/IndexTable.cs b/src/DataStreamSuite/DataStream/BlueOrb/IndexTable.cs new file mode 100644 index 000000000..72783a997 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/IndexTable.cs @@ -0,0 +1,93 @@ +namespace bts.BlueOrb; + +using BCG; + +using Bts.DataStreamCore; + +using Spark.Core.Collections; + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; + +public class IndexTable : IBlueRowCollection { + readonly IBlueRowCollection inputRows; + readonly IDictionary outputToInputIdMap; + readonly IDictionary inputToOutputIdMap; + readonly int idColIndex; + + public IndexTable(IBlueRowCollection inputRows, IDictionary outputToInputIdMap, IDictionary inputToOutputIdMap) { + this.inputRows = inputRows; + this.outputToInputIdMap = outputToInputIdMap; + this.inputToOutputIdMap = inputToOutputIdMap; + if (!inputRows.schema.columnInfo.TryGetFirst(c => c.colName == "id", out var idColInfo)) { + throw new ArgumentException("No 'id' column", nameof(inputRows)); + } + idColIndex = idColInfo.colNo; + } + + public IBlueTableSchema schema => inputRows.schema; + + public int Count => inputRows.Count; + + public IBlueRow this[int id] => + outputToInputIdMap.TryGetValue(id, out var inputId) + ? new IndexRow(inputRows[inputId], id, idColIndex, schema) + : throw new KeyNotFoundException(); + + IBlueRow IList.this[int index] { + get => GetRow(index); + set => throw new NotSupportedException(); + } + + public static IndexTable Create(IBlueRowCollection inputRows) { + var outputToInputIdMap = new Dictionary(); + var inputToOutputIdMap = new Dictionary(); + foreach (var r in inputRows) { + int inputId = r.Id(); + int outputId = outputToInputIdMap.Count + 1; + outputToInputIdMap[outputId] = inputId; + inputToOutputIdMap[inputId] = outputId; + } + return new(inputRows, outputToInputIdMap, inputToOutputIdMap); + } + + public static BlueTableImmutable CreateImmutable(IBlueTableSchema schema, IEnumerable rows) { + int idIndex = schema.ResolveIndices("id"); + var rowSetBd = ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer()).ToBuilder(); + foreach (var r in rows) { + int idx = rowSetBd.Count + 1; + var inputRow = (r as IndexRow)?.InputRow ?? r; + rowSetBd.Add(new(idx, new IndexRow(inputRow, idx, idIndex, schema))); + } + return new(schema, rowSetBd.ToImmutable()); + } + + public bool containsID(int id) => + inputRows.containsID(id); + + public IEnumerator GetEnumerator() { + foreach (var r in inputRows) { + yield return new IndexRow(r, inputToOutputIdMap[r.Id()], idColIndex, schema); + } + } + + public IBlueRow GetRow(int index) { + var r = inputRows.GetRow(index); + return new IndexRow(r, inputToOutputIdMap[r.Id()], idColIndex, schema); + } + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + if (outputToInputIdMap.TryGetValue(id, out var inputId)) { + var inputRow = inputRows[inputId]; + var r = new IndexRow(inputRow, id, idColIndex, schema); + if (BlueRowEqualityComparer.Equals(r, item)) { + var idx = inputRows.IndexOf(inputRow); + Log.assert(idx >= 0); + return idx; + } + } + return -1; + } +} diff --git a/src/DataStreamSuite/DataStream/DataStreamExtensions.cs b/src/DataStreamSuite/DataStream/DataStreamExtensions.cs index 10a8ecbed..e6100b048 100644 --- a/src/DataStreamSuite/DataStream/DataStreamExtensions.cs +++ b/src/DataStreamSuite/DataStream/DataStreamExtensions.cs @@ -7,6 +7,7 @@ using bts.api.core; using Bts.DataStreamCore; using Bts.PubSub; +using Bts.SharedStreams.StreamFactories; using mbts.utils; @@ -137,6 +138,21 @@ public static class DataStreamExtensions { public static ViewStream AsViewStream(this IObservable> source, IBlueTableSchema schema) where T : IBlueRow => source.AsDataPackets(schema).AsViewStream(); + public static ViewStream GroupByApply(this ViewStream viewStream, string[] columns, Func apply) { + var groupByTables = GroupByApplyStream.CreateGroupByTable(viewStream, columns); + var applyTables = GroupByApplyStream.CreateApplyTable(groupByTables, FunctionEvaluator.Create(apply)); + return applyTables.AsDataPackets().AsViewStream(); + } + + public static ViewStream GroupByApply(this ViewStream viewStream, string[] columns, Func apply) { + var groupByTables = GroupByApplyStream.CreateGroupByTable(viewStream, columns); + var applyTables = GroupByApplyStream.CreateApplyTable(groupByTables, new Func(ApplyWrapper)); + return applyTables.AsDataPackets().AsViewStream(); + + IBlueRowCollection? ApplyWrapper(EquatableKeyArrayWrapper groupKey, IBlueRowCollection groupRows) => + apply(groupKey.Values, groupRows); + } + public static IDisposable Subscribe(this ViewStream viewStream, Action onChange) { viewStream.onChange += OnChange; return Disposable.Create(Dispose); diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/EquatableKeyArrayWrapper.cs b/src/DataStreamSuite/DataStreamCore/BlueData/EquatableKeyArrayWrapper.cs new file mode 100644 index 000000000..2728cfb09 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/EquatableKeyArrayWrapper.cs @@ -0,0 +1,99 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using bts; + +using System; + +public readonly struct EquatableKeyArrayWrapper : IEquatable, IComparable { + readonly object?[] values; + + public EquatableKeyArrayWrapper(object?[] values) { + // This is an arbitrary max length, for performance reasons. We can change + // this (of course), but if we do, it should be deliberate. + Log.assert(values.Length < 128); + this.values = values; + } + + public object?[] Values => values; + + public int CompareTo(object? obj) { + if (obj is EquatableKeyArrayWrapper other) { + return CompareTo(other.values); + } + if (obj is object?[] objs) { + return CompareTo(objs); + } + if (obj is EquatableKeyWrapper otherKey) { + return CompareTo(otherKey.Value); + } + return 0; + } + + public int CompareTo(object?[] objs) { + for (int i = 0; i < objs.Length; i++) { + if (i == values.Length) { + return -1; + } + var value = values[i]; + var obj = objs[i]; + if (value is IComparable x) { + if (obj is EquatableKeyWrapper objWrapper) { + return x.CompareTo(objWrapper.Value); + } + return x.CompareTo(obj); + } + if (obj is IComparable y) { + return -y.CompareTo(value); + } + } + if (objs.Length == values.Length) { + return 0; + } + return 1; + } + + public bool Equals(EquatableKeyArrayWrapper other) => + Equals(other.values); + + public bool Equals(object?[] otherValues) { + if (values.Length != otherValues.Length) { + return false; + } + for (int i = 0; i < values.Length; i++) { + if (!Equals(values[i], otherValues[i])) { + return false; + } + } + return true; + } + + public override bool Equals(object? obj) { + if (obj is EquatableKeyArrayWrapper other) { + return Equals(other.values); + } + if (obj is object?[] objs) { + return Equals(objs); + } + if (obj is EquatableKeyWrapper otherKey) { + return Equals(new object?[] { otherKey.Value }); + } + return false; + } + + public override int GetHashCode() { + var hash = new HashCode(); + for (int i = 0; i < values.Length; i++) { + hash.Add(values[i]); + } + return hash.ToHashCode(); + } + + public static bool operator ==(EquatableKeyArrayWrapper left, EquatableKeyArrayWrapper right) { + return left.Equals(right); + } + + public static bool operator !=(EquatableKeyArrayWrapper left, EquatableKeyArrayWrapper right) { + return !(left == right); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/EquatableKeyWrapper.cs b/src/DataStreamSuite/DataStreamCore/BlueData/EquatableKeyWrapper.cs new file mode 100644 index 000000000..84660c7ae --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/EquatableKeyWrapper.cs @@ -0,0 +1,41 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using System; + +public readonly struct EquatableKeyWrapper : IEquatable, IComparable { + public readonly object Value; + + public EquatableKeyWrapper(object value) => + Value = value; + + public int CompareTo(object? obj) { + if (Value is IComparable x) { + if (obj is EquatableKeyWrapper objWrapper) { + return x.CompareTo(objWrapper.Value); + } + return x.CompareTo(obj); + } + if (obj is IComparable y) { + return -y.CompareTo(Value); + } + return 0; + } + + public bool Equals(EquatableKeyWrapper other) => + Equals(Value, other.Value); + + public override bool Equals(object? obj) => + obj is EquatableKeyWrapper other ? Equals(other) : Equals(Value, obj); + + public override int GetHashCode() => + Value.GetHashCode(); + + public static bool operator ==(EquatableKeyWrapper left, EquatableKeyWrapper right) { + return left.Equals(right); + } + + public static bool operator !=(EquatableKeyWrapper left, EquatableKeyWrapper right) { + return !(left == right); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/SingleRowTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/SingleRowTable.cs new file mode 100644 index 000000000..4c0bd23c8 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/SingleRowTable.cs @@ -0,0 +1,35 @@ +namespace bts.BlueOrb; + +using BCG; + +using Bts.DataStreamCore; + +using System; +using System.Collections.Generic; + +public class SingleRowTable : IBlueRowCollection { + readonly IBlueRow row; + + public SingleRowTable(IBlueRow row) => + this.row = row; + + public IBlueRow this[int id] => + id == row.Id() ? row : throw new KeyNotFoundException(); + + public IBlueTableSchema schema => row.schema; + + public int Count => 1; + + public bool containsID(int id) => + id == row.Id(); + + public IEnumerator GetEnumerator() { + yield return row; + } + + public IBlueRow GetRow(int index) => + index == 0 ? row : throw new IndexOutOfRangeException(); + + public int IndexOf(IBlueRow item) => + BlueRowEqualityComparer.Equals(item, row) ? 0 : -1; +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/TableUpdateGroupingRowCollection.cs b/src/DataStreamSuite/DataStreamCore/BlueData/TableUpdateGroupingRowCollection.cs new file mode 100644 index 000000000..a40490b06 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/TableUpdateGroupingRowCollection.cs @@ -0,0 +1,109 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using bts.api.core; +using bts.collections; + +using Spark.Core.Collections; + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; + +public class TableUpdateGroupingRowCollection : IBlueRowCollection, IDictionary where TGroupKey : IEquatable { + readonly TableUpdateGrouping table; + + public TableUpdateGroupingRowCollection(TableUpdateGrouping table, IBlueTableSchema schema) { + this.table = table; + Schema = schema; + } + + public IBlueTableSchema Schema { get; } + + public int Count => table.table.state.Count; + + public ICollection Keys => + ReadOnlyDictionaryKeyCollection.create(table.table.state); + + public ICollection Values => + ReadOnlyDictionaryValueCollection.create(table.table.state); + + public bool IsReadOnly => true; + + IBlueTableSchema IBlueRowCollection.schema => Schema; + + public IBlueRow this[int id] { + get => table.table.state[id]; + set => throw new NotSupportedException(); + } + + IBlueRow IList.this[int index] { + get => table.table.GetValueAtIndex(index); + set => throw new NotSupportedException(); + } + + public IBlueRow GetRow(int index) => + table.table.GetValueAtIndex(index); + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + int index = table.table.BinarySearch(id); + if (index >= 0) { + var row = table.table.GetValueAtIndex(index); + if (BlueRowEqualityComparer.Equals(row, item)) { + return index; + } + } + return -1; + } + + public IList GetColumn(int id) => + table.table.AsList().WrapSelector(r => r[id], null); + + public bool ContainsKey(int key) => + table.table.state.ContainsKey(key); + + public bool Contains(KeyValuePair item) => + table.table.state.TryGetValue(item.Key, out var r) + && r.Values.Cast().SequenceEqual(item.Value.Cast()); + + public void CopyTo(KeyValuePair[] array, int arrayIndex) { + foreach (var kv in table.table.state) { + array[arrayIndex++] = kv; + } + } + + public bool TryGetValue(int key, [MaybeNullWhen(false)] out IBlueRow value) => + table.table.state.TryGetValue(key, out value); + + public IEnumerator GetEnumerator() => + table.table.state.Values.GetEnumerator(); + + bool IBlueRowCollection.containsID(int id) => + ContainsKey(id); + + IEnumerator IEnumerable.GetEnumerator() => + table.table.state.Values.GetEnumerator(); + + void IDictionary.Add(int key, IBlueRow value) => + throw new NotSupportedException(); + + bool IDictionary.Remove(int key) => + throw new NotImplementedException(); + + void ICollection>.Add(KeyValuePair item) => + throw new NotSupportedException(); + + void ICollection>.Clear() => + throw new NotImplementedException(); + + bool ICollection>.Remove(KeyValuePair item) => + throw new NotSupportedException(); + + IEnumerator> IEnumerable>.GetEnumerator() => + table.table.state.GetEnumerator(); +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamFactory.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamFactory.cs index d00214014..cb441e9dc 100644 --- a/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamFactory.cs +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/DataStreamFactory.cs @@ -63,4 +63,16 @@ public static class DataStreamFactoryExtensions { var sf = new SingleDataStreamFactory(meta, singleStreamfactory, logger); streamFactory.Add(sf); } + + public static void Add(this DataStreamFactory streamFactory, string[] names, Func singleStreamfactory, ILogger? logger = null, StreamFactoryArgStore.TryGetParamsDelegate? tryGetParams = null) { + if (names.Length == 0) { + throw new ArgumentException("StreamFactory must have at least one name", nameof(names)); + } + var meta = new DataStreamMetadata() { Name = names[0], AlternateNames = names.Skip(1).ToList() }; + var sf = new SingleDataStreamFactory(meta, singleStreamfactory, logger, tryGetParams); + streamFactory.Add(sf); + } + + public static ViewStream? Create(this IDataStreamFactory streamFactory, string streamId, params object?[] args) => + streamFactory.Create(streamId, args.Select((a, i) => ((string?)null, (int?)i, a))); } diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/ExtensionFunctionEvaluator.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/ExtensionFunctionEvaluator.cs new file mode 100644 index 000000000..8f9aaeadd --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/ExtensionFunctionEvaluator.cs @@ -0,0 +1,294 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using bts.collections; + +using System; +using System.Collections.Generic; +using System.Linq; + +public class ExtensionFunctionEvaluator : IExtensionFunctionEvaluator { + readonly Dictionary, object?> Evaluate)>> evaluators = new(); + + public IEnumerable SupportedFunctions => + evaluators.Keys; + + public object? Evaluate(string functionId, T target, IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (!evaluators.TryGetValue(functionId, out var fnEvals)) { + throw new ArgumentException($"Failed to find evaluator for function '{functionId}'", nameof(functionId)); + } + var argList = args.ToList(); + var argSig = argList.WrapSelector(a => (a.Name, a.Value?.GetType() ?? typeof(object)), null); + foreach (var (sig, eval) in fnEvals) { + if (IsSignatureMatch(sig)) { + return eval(target, argList); + } + } + throw new ArgumentException($"Failed to find evaluator for function '{functionId}' with given args", nameof(args)); + + bool IsSignatureMatch((string? Name, Type? Type)[] sig) { + if (argSig.Count != sig.Length) { + return false; + } + for (int i = 0; i < sig.Length; i++) { + var p = sig[i]; + var a = argList[i]; + if (!IsNameMatch(p.Name, a.Name) || !IsTypeMatch(p.Type, a.Value?.GetType())) { + return false; + } + } + return true; + + bool IsNameMatch(string? pName, string? aName) => + string.IsNullOrEmpty(pName) || string.IsNullOrEmpty(aName) || string.Equals(pName, aName, StringComparison.CurrentCultureIgnoreCase); + + bool IsTypeMatch(Type? pType, Type? aType) { + if (pType is null) { + return true; + } + if (aType is null) { + return !pType.IsValueType; + } + return pType.IsAssignableFrom(aType); + } + } + } + + public void AddOverload(string functionId, Func evaluate) { + var sig = Array.Empty<(string?, Type?)>(); + evaluators.GetOrAdd(functionId, _ => new(1)).Add((sig, EvaluateWrapper)); + + object? EvaluateWrapper(T target, List<(string?, int?, object?)> args) => + evaluate(target); + } + + public void AddOverload(string functionId, Func evaluate) { + var sig = new (string?, Type?)[] { (null, typeof(T1)) }; + evaluators.GetOrAdd(functionId, _ => new(1)).Add((sig, EvaluateWrapper)); + + object? EvaluateWrapper(T target, List<(string?, int?, object?)> argList) { + if (argList.Count != 1) { + throw new ArgumentException($"Expected exactly 1 args (got {argList.Count})", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[0].Item3, out T1 arg1)) { + throw new ArgumentException($"Failed to convert first arg to type '{typeof(T1)}'", nameof(argList)); + } + return evaluate(target, arg1); + } + } + + public void AddOverload(string functionId, Func evaluate) { + var sig = new (string?, Type?)[] { (null, typeof(T1)), (null, typeof(T2)) }; + evaluators.GetOrAdd(functionId, _ => new(1)).Add((sig, EvaluateWrapper)); + + object? EvaluateWrapper(T target, List<(string?, int?, object?)> argList) { + if (argList.Count != 2) { + throw new ArgumentException($"Expected exactly 2 args (got {argList.Count})", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[0].Item3, out T1 arg1)) { + throw new ArgumentException($"Failed to convert first arg to type '{typeof(T1)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[1].Item3, out T2 arg2)) { + throw new ArgumentException($"Failed to convert second arg to type '{typeof(T2)}'", nameof(argList)); + } + return evaluate(target, arg1, arg2); + } + } + + public void AddOverload(string functionId, Func evaluate) { + var sig = new (string?, Type?)[] { (null, typeof(T1)), (null, typeof(T2)), (null, typeof(T3)) }; + evaluators.GetOrAdd(functionId, _ => new(1)).Add((sig, EvaluateWrapper)); + + object? EvaluateWrapper(T target, List<(string?, int?, object?)> argList) { + if (argList.Count != 3) { + throw new ArgumentException($"Expected exactly 3 args (got {argList.Count})", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[0].Item3, out T1 arg1)) { + throw new ArgumentException($"Failed to convert first arg to type '{typeof(T1)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[1].Item3, out T2 arg2)) { + throw new ArgumentException($"Failed to convert second arg to type '{typeof(T2)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[2].Item3, out T3 arg3)) { + throw new ArgumentException($"Failed to convert third arg to type '{typeof(T3)}'", nameof(argList)); + } + return evaluate(target, arg1, arg2, arg3); + } + } + + public void AddOverload(string functionId, Func evaluate) { + var sig = new (string?, Type?)[] { + (null, typeof(T1)), + (null, typeof(T2)), + (null, typeof(T3)), + (null, typeof(T4)), + }; + evaluators.GetOrAdd(functionId, _ => new(1)).Add((sig, EvaluateWrapper)); + + object? EvaluateWrapper(T target, List<(string?, int?, object?)> argList) { + if (argList.Count != 4) { + throw new ArgumentException($"Expected exactly 4 args (got {argList.Count})", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[0].Item3, out T1 arg1)) { + throw new ArgumentException($"Failed to convert first arg to type '{typeof(T1)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[1].Item3, out T2 arg2)) { + throw new ArgumentException($"Failed to convert second arg to type '{typeof(T2)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[2].Item3, out T3 arg3)) { + throw new ArgumentException($"Failed to convert third arg to type '{typeof(T3)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[3].Item3, out T4 arg4)) { + throw new ArgumentException($"Failed to convert fourth arg to type '{typeof(T4)}'", nameof(argList)); + } + return evaluate(target, arg1, arg2, arg3, arg4); + } + } + + public void AddOverload(string functionId, Func evaluate) { + var sig = new (string?, Type?)[] { + (null, typeof(T1)), + (null, typeof(T2)), + (null, typeof(T3)), + (null, typeof(T4)), + (null, typeof(T5)), + }; + evaluators.GetOrAdd(functionId, _ => new(1)).Add((sig, EvaluateWrapper)); + + object? EvaluateWrapper(T target, List<(string?, int?, object?)> argList) { + if (argList.Count != 5) { + throw new ArgumentException($"Expected exactly 5 args (got {argList.Count})", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[0].Item3, out T1 arg1)) { + throw new ArgumentException($"Failed to convert first arg to type '{typeof(T1)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[1].Item3, out T2 arg2)) { + throw new ArgumentException($"Failed to convert second arg to type '{typeof(T2)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[2].Item3, out T3 arg3)) { + throw new ArgumentException($"Failed to convert third arg to type '{typeof(T3)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[3].Item3, out T4 arg4)) { + throw new ArgumentException($"Failed to convert fourth arg to type '{typeof(T4)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[4].Item3, out T5 arg5)) { + throw new ArgumentException($"Failed to convert fifth arg to type '{typeof(T5)}'", nameof(argList)); + } + return evaluate(target, arg1, arg2, arg3, arg4, arg5); + } + } + + public void AddOverload(string functionId, Func evaluate) { + var sig = new (string?, Type?)[] { + (null, typeof(T1)), + (null, typeof(T2)), + (null, typeof(T3)), + (null, typeof(T4)), + (null, typeof(T5)), + (null, typeof(T6)), + }; + evaluators.GetOrAdd(functionId, _ => new(1)).Add((sig, EvaluateWrapper)); + + object? EvaluateWrapper(T target, List<(string?, int?, object?)> argList) { + if (argList.Count != 6) { + throw new ArgumentException($"Expected exactly 6 args (got {argList.Count})", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[0].Item3, out T1 arg1)) { + throw new ArgumentException($"Failed to convert first arg to type '{typeof(T1)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[1].Item3, out T2 arg2)) { + throw new ArgumentException($"Failed to convert second arg to type '{typeof(T2)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[2].Item3, out T3 arg3)) { + throw new ArgumentException($"Failed to convert third arg to type '{typeof(T3)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[3].Item3, out T4 arg4)) { + throw new ArgumentException($"Failed to convert fourth arg to type '{typeof(T4)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[4].Item3, out T5 arg5)) { + throw new ArgumentException($"Failed to convert fifth arg to type '{typeof(T5)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[5].Item3, out T6 arg6)) { + throw new ArgumentException($"Failed to convert sixth arg to type '{typeof(T6)}'", nameof(argList)); + } + return evaluate(target, arg1, arg2, arg3, arg4, arg5, arg6); + } + } + + public void AddOverload(string functionId, Func evaluate) { + var sig = new (string?, Type?)[] { + (null, typeof(T1)), + (null, typeof(T2)), + (null, typeof(T3)), + (null, typeof(T4)), + (null, typeof(T5)), + (null, typeof(T6)), + (null, typeof(T7)), + }; + evaluators.GetOrAdd(functionId, _ => new(1)).Add((sig, EvaluateWrapper)); + + object? EvaluateWrapper(T target, List<(string?, int?, object?)> argList) { + if (argList.Count != 7) { + throw new ArgumentException($"Expected exactly 7 args (got {argList.Count})", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[0].Item3, out T1 arg1)) { + throw new ArgumentException($"Failed to convert first arg to type '{typeof(T1)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[1].Item3, out T2 arg2)) { + throw new ArgumentException($"Failed to convert second arg to type '{typeof(T2)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[2].Item3, out T3 arg3)) { + throw new ArgumentException($"Failed to convert third arg to type '{typeof(T3)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[3].Item3, out T4 arg4)) { + throw new ArgumentException($"Failed to convert fourth arg to type '{typeof(T4)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[4].Item3, out T5 arg5)) { + throw new ArgumentException($"Failed to convert fifth arg to type '{typeof(T5)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[5].Item3, out T6 arg6)) { + throw new ArgumentException($"Failed to convert sixth arg to type '{typeof(T6)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[6].Item3, out T7 arg7)) { + throw new ArgumentException($"Failed to convert seventh arg to type '{typeof(T7)}'", nameof(argList)); + } + return evaluate(target, arg1, arg2, arg3, arg4, arg5, arg6, arg7); + } + } + + public void AddOverload(string functionId, Func evaluate) { + var sig = new (string?, Type?)[] { (null, typeof(T1)), (null, typeof(T2)), (null, typeof(T3)) }; + evaluators.GetOrAdd(functionId, _ => new(1)).Add((sig, EvaluateWrapper)); + + object? EvaluateWrapper(T target, List<(string?, int?, object?)> argList) { + if (argList.Count != 8) { + throw new ArgumentException($"Expected exactly 8 args (got {argList.Count})", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[0].Item3, out T1 arg1)) { + throw new ArgumentException($"Failed to convert first arg to type '{typeof(T1)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[1].Item3, out T2 arg2)) { + throw new ArgumentException($"Failed to convert second arg to type '{typeof(T2)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[2].Item3, out T3 arg3)) { + throw new ArgumentException($"Failed to convert third arg to type '{typeof(T3)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[3].Item3, out T4 arg4)) { + throw new ArgumentException($"Failed to convert fourth arg to type '{typeof(T4)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[4].Item3, out T5 arg5)) { + throw new ArgumentException($"Failed to convert fifth arg to type '{typeof(T5)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[5].Item3, out T6 arg6)) { + throw new ArgumentException($"Failed to convert sixth arg to type '{typeof(T6)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[6].Item3, out T7 arg7)) { + throw new ArgumentException($"Failed to convert seventh arg to type '{typeof(T7)}'", nameof(argList)); + } + if (!ConvertUtils.try_convert(argList[7].Item3, out T8 arg8)) { + throw new ArgumentException($"Failed to convert eighth arg to type '{typeof(T8)}'", nameof(argList)); + } + return evaluate(target, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/IdentityFunctionEvaluator.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/IdentityFunctionEvaluator.cs new file mode 100644 index 000000000..9af5c4dd0 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/IdentityFunctionEvaluator.cs @@ -0,0 +1,16 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using System.Collections.Generic; +using System.Linq; + +public class IdentityFunctionEvaluator : IFunctionEvaluator { + public object? Evaluate(string functionId, IEnumerable<(string? Name, int? Index, object? Value)> args) { + var argList = args.Select(a => a.Value).ToList(); + return argList.Count switch { + 0 => null, + 1 => argList[0], + _ => argList, + }; + } +} diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs index 3e49d4e02..c5145e581 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs @@ -63,5 +63,6 @@ static class Program { .AddPubTableBroker() .AddRemoteQueryStreams() .AddPrimitiveStreams(context.Configuration.GetSection("PrimitiveStream")) + .AddGroupByApplyStream() .AddMarketDataStreams(); } diff --git a/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs b/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs index c33deccbc..3b5363d08 100644 --- a/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs +++ b/src/DataStreamSuite/SharedStreams/Services/SharedStreamsServices.cs @@ -25,6 +25,9 @@ public static class SharedStreamsServices { public static IServiceCollection AddParquetReaderStream(this IServiceCollection services) => services.addStreamFactory(); + public static IServiceCollection AddGroupByApplyStream(this IServiceCollection services) => + services.addStreamFactory(); + static IServiceCollection addStreamFactory(this IServiceCollection services) where T : class, IDataStreamFactory => services.AddSingleton() .AddSingleton(sp => sp.GetRequiredService()); diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/GroupByApplyStreamFactory.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/GroupByApplyStreamFactory.cs new file mode 100644 index 000000000..a28eb77f2 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/GroupByApplyStreamFactory.cs @@ -0,0 +1,163 @@ +namespace Bts.SharedStreams.StreamFactories; + +using BCG; + +using bts; +using bts.api.core; +using bts.BlueOrb; + +using Bts.DataStream; +using Bts.DataStreamCore; + +using Microsoft.Extensions.Logging; + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Reactive.Linq; + +public class GroupByApplyStreamFactory : IDataStreamFactory { + readonly ILogger logger; + + public GroupByApplyStreamFactory(ILogger logger) => + this.logger = logger; + + public IObservable> CreateGroupByApplyTable(ViewStream inputStream, IFunctionEvaluator groupBy, IFunctionEvaluator apply) => + CreateApplyTable(CreateGroupByTable(inputStream, groupBy), apply); + + public IObservable> CreateApplyTable(IObservable, EquatableKeyWrapper>> inputStream, IFunctionEvaluator apply) { + ImmutableDictionary groupTables = ImmutableDictionary.Empty; + ImmutableDictionary inputToOutputIds = ImmutableDictionary.Empty; + ImmutableDictionary outputToInputIds = ImmutableDictionary.Empty; + int idCounter = 0; + + return Observable.Create>(Subscribe); + + int GetNextId() => + Interlocked.Increment(ref idCounter); + + IDisposable Subscribe(IObserver> observer) { + int hashCounter = 0; + var hashIndex = new Dictionary(); + var tableOut = TableUpdate.mk_empty(r => r.Id()); + return inputStream.Subscribe(OnNext, OnError, OnCompleted); + + void OnNext(TableUpdate, EquatableKeyWrapper> upd) { + var bd = tableOut.to_builder(); + foreach (var group in upd.removed) { + var hash = hashIndex[group.group_key]; + bd.remove(hash); + } + var groupTablesBd = groupTables.ToBuilder(); + foreach (var group in upd.added.Concat(upd.changed)) { + Log.assert(group.table.state.Count > 0); + var hash = hashIndex.GetOrAdd(group.group_key, _ => hashCounter++); + var groupRowCollection = new TableUpdateGroupingRowCollection(group, group.table.state.Values.First().schema); + var applyArgs = new (string?, int?, object?)[] { ("rows", 0, groupRowCollection) }; + var res = apply.Evaluate(string.Empty, applyArgs); + if (res is IBlueRow resSingleRow) { + UpdateGroupTable(group.group_key, new SingleRowTable(resSingleRow)); + } else if (res is IBlueRowCollection resRows) { + UpdateGroupTable(group.group_key, resRows); + } else { + throw new InvalidOperationException("Invalid result returned by 'Apply' fn"); + } + } + groupTables = groupTablesBd.ToImmutable(); + tableOut = bd.to_table(); + observer.OnNext(tableOut); + + void UpdateGroupTable(EquatableKeyWrapper groupKey, IBlueRowCollection groupRows) { + var indexedRows = groupTablesBd.GetOrAdd(groupKey, _ => new(GetNextId)).Reset(groupRows); + foreach (var r in indexedRows) { + bd.add_or_update(r); + } + } + } + } + + void OnError(Exception ex) => + Log.DataFlow.error($$"""[stream_err.json]{ "typ": "{{GetType().Name}}", "inpTyp": {{inputStream.GetType()}}, "err": "{{ex}}" }"""); + + void OnCompleted() => + Log.DataFlow.warn($$"""[stream_fin.json]{ "typ": "{{GetType().Name}}", "inpTyp": "{{inputStream.GetType()}}" }"""); + } + + public IObservable, EquatableKeyWrapper>> CreateGroupByTable(ViewStream inputStream, IFunctionEvaluator groupBy) { + return Observable.Create, EquatableKeyWrapper>>(Subscribe); + + IDisposable Subscribe(IObserver, EquatableKeyWrapper>> observer) { + TableUpdate, EquatableKeyWrapper> tableOut = TableUpdate, EquatableKeyWrapper>.mk_empty(r => r.group_key); + return inputStream.AsObservable().AsTables().GroupTable(r => r.Id(), GroupKey).Subscribe(observer); + + EquatableKeyWrapper GroupKey(IBlueRow r) { + var groupByArgs = new (string?, int?, object?)[] { ("row", 0, r) }; + var groupKey = new EquatableKeyWrapper(groupBy.Evaluate(string.Empty, groupByArgs) ?? DBNull.Value); + return groupKey; + } + } + } + + public IObservable? CreateGroupByApplyStream(GroupByApplyStreamParams args) { + ArgumentNullException.ThrowIfNull(args.InputStream); + var groupFn = args.GroupFunction ?? new IdentityFunctionEvaluator(); + var applyFn = args.ApplyFunction ?? new IdentityFunctionEvaluator(); + return CreateGroupByApplyTable(args.InputStream, groupFn, applyFn).AsDataPackets(); + } + + public ViewStream? Create(string streamId, IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (SupportedStreams.GroupByApply.IsMatch(streamId)) { + return createGroupByApply(args); + } + return null; + } + + public IEnumerable GetSupportedStreams() { + yield return SupportedStreams.GroupByApply; + } + + ViewStream? createGroupByApply(IEnumerable<(string? Name, int? Index, object? Value)> args) { + if (!StreamFactoryArgStore.TryGetParams(args, out GroupByApplyStreamParams? gbasp, out var err)) { + logger.LogInvalidParams("groupByApply", err); + return null; + } + return CreateGroupByApplyStream(gbasp)?.AsViewStream(); + } + + static class SupportedStreams { + public static readonly DataStreamMetadata GroupByApply = new() { + Name = "groupByApply" + }; + } + + class GroupTableBuilder { + readonly Func getUniqueId; + ImmutableDictionary inputToOutputIds = ImmutableDictionary.Empty; + ImmutableDictionary outputToInputIds = ImmutableDictionary.Empty; + IndexTable? current; + + public GroupTableBuilder(Func getUniqueId) => + this.getUniqueId = getUniqueId; + + public IndexTable? Current => current; + + public IndexTable Reset(IBlueRowCollection rows) { + var inputToOutputIdsBd = inputToOutputIds.ToBuilder(); + var outputToInputIdsBd = outputToInputIds.ToBuilder(); + foreach (var r in rows) { + int inputId = r.Id(); + if (!inputToOutputIdsBd.TryGetValue(inputId, out var outputId)) { + outputId = getUniqueId(); + inputToOutputIdsBd.Add(inputId, outputId); + outputToInputIdsBd.Add(outputId, inputId); + } + Log.assert(outputToInputIdsBd.TryGetValue(outputId, out var oinputId) && oinputId == inputId); + Log.assert(inputToOutputIdsBd.TryGetValue(inputId, out var ioutputId) && ioutputId == outputId); + } + inputToOutputIds = inputToOutputIdsBd.ToImmutable(); + outputToInputIds = outputToInputIdsBd.ToImmutable(); + current = new(rows, outputToInputIds, inputToOutputIds); + return current; + } + } +} diff --git a/src/DataStreamSuite/SharedStreams/StreamFactories/GroupByApplyStreamParams.cs b/src/DataStreamSuite/SharedStreams/StreamFactories/GroupByApplyStreamParams.cs new file mode 100644 index 000000000..af6d35928 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/StreamFactories/GroupByApplyStreamParams.cs @@ -0,0 +1,22 @@ +namespace Bts.SharedStreams.StreamFactories; + +using BCG; + +using Bts.DataStreamCore; + +public class GroupByApplyStreamParams { + [StreamArgument(RegexNames = new[] { "inputStream" }, Position = 0)] + public ViewStream? InputStream { get; set; } + + [StreamArgument("group", Position = 1)] + public IFunctionEvaluator? GroupFunction { get; set; } + + [StreamArgument("apply", Position = 2)] + public IFunctionEvaluator? ApplyFunction { get; set; } + + public void Validate() { + if (GroupFunction is null) { + + } + } +} diff --git a/src/client/spark_core/Collections/ReadOnlyDictionaryKeyCollection.cs b/src/client/spark_core/Collections/ReadOnlyDictionaryKeyCollection.cs new file mode 100644 index 000000000..e2b1d7a7e --- /dev/null +++ b/src/client/spark_core/Collections/ReadOnlyDictionaryKeyCollection.cs @@ -0,0 +1,45 @@ +#nullable enable +namespace Spark.Core.Collections; + +using System; +using System.Collections; +using System.Collections.Generic; + +public static class ReadOnlyDictionaryKeyCollection { + public static ReadOnlyDictionaryKeyCollection create(IReadOnlyDictionary source) where TKey : notnull => new(source); +} + +public class ReadOnlyDictionaryKeyCollection : ICollection where TKey : notnull { + readonly IReadOnlyDictionary _source; + + public ReadOnlyDictionaryKeyCollection(IReadOnlyDictionary source) => + _source = source; + + public int Count => _source.Count; + + public bool IsReadOnly => true; + + public void Add(TKey item) => + throw new NotSupportedException(); + + public void Clear() => + throw new NotSupportedException(); + + public bool Contains(TKey item) => + _source.ContainsKey(item); + + public void CopyTo(TKey[] array, int array_index) { + foreach (var k in _source.Keys) { + array[array_index++] = k; + } + } + + public IEnumerator GetEnumerator() => + _source.Keys.GetEnumerator(); + + public bool Remove(TKey item) => + throw new NotSupportedException(); + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); +} diff --git a/src/client/spark_core/Collections/ReadOnlyDictionaryValueCollection.cs b/src/client/spark_core/Collections/ReadOnlyDictionaryValueCollection.cs new file mode 100644 index 000000000..3ce451112 --- /dev/null +++ b/src/client/spark_core/Collections/ReadOnlyDictionaryValueCollection.cs @@ -0,0 +1,47 @@ +#nullable enable +namespace Spark.Core.Collections; + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +public static class ReadOnlyDictionaryValueCollection { + public static ReadOnlyDictionaryValueCollection create(IReadOnlyDictionary source) where TKey : notnull => + new(source); +} + +public class ReadOnlyDictionaryValueCollection : ICollection where TKey : notnull { + readonly IReadOnlyDictionary _source; + + public ReadOnlyDictionaryValueCollection(IReadOnlyDictionary source) => + _source = source; + + public int Count => _source.Count; + + public bool IsReadOnly => true; + + public void Add(TValue item) => + throw new NotSupportedException(); + + public void Clear() => + throw new NotSupportedException(); + + public bool Contains(TValue item) => + _source.Values.Contains(item); + + public void CopyTo(TValue[] array, int array_index) { + foreach (var k in _source.Values) { + array[array_index++] = k; + } + } + + public IEnumerator GetEnumerator() => + _source.Values.GetEnumerator(); + + public bool Remove(TValue item) => + throw new NotSupportedException(); + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); +} diff --git a/src/client/spark_core/Update/TableUpdate.cs b/src/client/spark_core/Update/TableUpdate.cs index 69a912e59..29f1d885c 100644 --- a/src/client/spark_core/Update/TableUpdate.cs +++ b/src/client/spark_core/Update/TableUpdate.cs @@ -51,6 +51,12 @@ namespace bts.api.core { public Func? key_extractor { get; private set; } + public TKey GetKeyAtIndex(int index) => + _state[index].Key; + + public T GetValueAtIndex(int index) => + _state[index].Value; + public static TableUpdate mk_empty(Func key_extractor) => new( ImmutableSortedSet>.Empty, Array.Empty(), @@ -61,6 +67,11 @@ namespace bts.api.core { public IList AsList() => _state.WrapSelector(kv => kv.Value, null); + public int BinarySearch(TKey key) => +#nullable disable + _state.IndexOf(new(key, default)); +#nullable enable + public IList> AsKeyValueList() => _state; @@ -311,6 +322,10 @@ namespace bts.api.core { return !_removed.ContainsKey(key) && _state.state.TryGetValue(key, out result); } + [return: MaybeNull] + public T GetValueOrDefault(TKey key) => + TryGetValue(key, out var res) ? res : default; + /// /// Creates a TableUpdate instance from the current state ///