cleanup: Fix a ton of build warnings (cs)
All these warnings have been bothering me for a while. The work isn't done, but this commit chops them down from ~1000 to ~600. * AppendCollection.cs * AppendStreamModified.cs * BenzingaNewsStream.cs * BenzingaNewsSymbolStateStream.cs * BookStream.cs * CSVSaverStream.cs * ChartsUtils.cs * ColumnInfo.cs * ContractUtils.cs * ContractUtils.cs * DiffStream.cs * FuturesActiveContractStream.cs * FuzzyJoinOverTimeStream.cs * HistoricalBarsStream.cs * HistoricalBarsStream.cs * InstrumentsStream.cs * JoinOperandRowIndex.cs * JoinRow.cs * JoinStream.cs * JoinTable.cs * JoinTableSchema.cs * LatencyStream.cs * LeftInternalJoinStream.cs * LightForeignJoinRow.cs * LightForeignJoinRowList.cs * LightLeftForeignJoinRow.cs * LiteralStream.cs * NLStream.cs * NLToDSLStream.cs * NLToIndexedFewShotPromptStream.cs * NonIncGroupByStream.cs * NonIncJoinBookkeeper.cs * NonIncJoinSpec.cs * OHLCVStream.cs * OpenAIPromptStream.cs * PriceStatsStream.cs * PriceStatsStream.cs * ProjectHelper.cs * ProjectStream.cs * PubSubServerStateStream.cs * PubsubHeartbeatStream.cs * SimpleNode.cs * SortRow.cs * SpatialArbStatsStream.cs * SpatialFuturePairsStream.cs * SpreadPerformanceEvaluator.cs * SpreadPerformanceStream.cs * StreamIOCommon.cs * SuperJoinStream.cs * TableMetaDataStream.cs * TextWriterStream.cs * TimeAverageStream.cs
This commit is contained in:
1 parent
7418f5b5a1
commit
a4efaa0089
48 files changed
+610
-590
No files matched your search
@@ -1,5 +1,4 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using mbts;
|
||||
|
||||
@@ -12,30 +11,21 @@ namespace BCG
|
||||
{
|
||||
ViewStream myLStream, myRStream;
|
||||
|
||||
DataPacket myLastR = null, myLastL = null;
|
||||
DataPacket? myLastR = null, myLastL = null;
|
||||
|
||||
BlueTable myLastFired = null;
|
||||
BlueTable? myLastFired = null;
|
||||
|
||||
double myThreshold = .00005;
|
||||
|
||||
string[] myColumnsToMatch;
|
||||
public DiffStream(ViewStream v1, ViewStream v2) : this(null, v1, v2, "id") { }
|
||||
public DiffStream(ViewStream v1, ViewStream v2, params string[] ColumnsToMatch) : this(null, v1, v2, ColumnsToMatch) { }
|
||||
public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) :
|
||||
this(null, v1, v2, thresh, ColumnsToMatch) { }
|
||||
|
||||
public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
|
||||
{
|
||||
public DiffStream(ViewStream v1, ViewStream v2) : this(v1, v2, "id") { }
|
||||
public DiffStream(ViewStream v1, ViewStream v2, params string[] columnsToMatch) : this(v1, v2, .00005, columnsToMatch) { }
|
||||
public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch) {
|
||||
myThreshold = thresh;
|
||||
myLStream = v1;
|
||||
myRStream = v2;
|
||||
myColumnsToMatch = columnsToMatch;
|
||||
}
|
||||
public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
|
||||
: this(db,v1,v2,.00005,columnsToMatch)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
@@ -52,8 +42,14 @@ namespace BCG
|
||||
|
||||
struct ListComparer : IEqualityComparer<List<object>>
|
||||
{
|
||||
public bool Equals(List<object> x, List<object> y)
|
||||
public bool Equals(List<object>? x, List<object>? y)
|
||||
{
|
||||
if (x is null) {
|
||||
return y is null;
|
||||
}
|
||||
if (y is null) {
|
||||
return false;
|
||||
}
|
||||
if (x.Count != y.Count)
|
||||
return false;
|
||||
|
||||
@@ -74,21 +70,20 @@ namespace BCG
|
||||
|
||||
void myHandleCallbacks()
|
||||
{
|
||||
DataPacket packet = null;
|
||||
|
||||
DataPacket? packet;
|
||||
if (myLastL == null || myLastR == null)
|
||||
{
|
||||
packet = myLastL == null ? myLastR : myLastL;
|
||||
if (packet != null)
|
||||
if (packet != null) {
|
||||
myFireEvent(packet);
|
||||
|
||||
myLastFired = new BlueTable(packet.newRows);
|
||||
myLastFired = new BlueTable(packet.newRows);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
BlueTable table = new BlueTable(myLastL.newRows.schema);
|
||||
|
||||
Dictionary<List<object>, Pair<BlueRow, BlueRow>> rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(new ListComparer());
|
||||
Dictionary<List<object>, Pair<BlueRow?, BlueRow?>> rowPairs = new(new ListComparer());
|
||||
|
||||
foreach(BlueRow row in myLastL.newRows)
|
||||
{
|
||||
@@ -104,7 +99,7 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
rowPairs[hash] = new Pair<BlueRow, BlueRow>(row, null);
|
||||
rowPairs[hash] = new(row, null);
|
||||
}
|
||||
|
||||
foreach (BlueRow row in myLastR.newRows)
|
||||
@@ -119,12 +114,11 @@ namespace BCG
|
||||
catch (Exception) { }
|
||||
}
|
||||
|
||||
Pair<BlueRow, BlueRow> pair;
|
||||
rowPairs.TryGetValue(hash, out pair);
|
||||
rowPairs.TryGetValue(hash, out var pair);
|
||||
|
||||
|
||||
if (pair == null)
|
||||
pair = new Pair<BlueRow, BlueRow>(null, row);
|
||||
pair = new(null, row);
|
||||
else
|
||||
pair.second = row;
|
||||
|
||||
@@ -132,13 +126,11 @@ namespace BCG
|
||||
}
|
||||
|
||||
int id = 0;
|
||||
foreach (KeyValuePair<List<object>, Pair<BlueRow, BlueRow>> hashpair in rowPairs)
|
||||
foreach (var (hashVals, pair) in rowPairs)
|
||||
{
|
||||
Pair<BlueRow, BlueRow> pair = hashpair.Value;
|
||||
|
||||
if (pair.first == null || pair.second == null)
|
||||
{
|
||||
BlueRow row = pair.first == null ? pair.second : pair.first;
|
||||
var row = pair.first ?? pair.second ?? throw new InvalidOperationException("Both rows in pair are null");
|
||||
|
||||
Dictionary<string, object> newRow = new Dictionary<string, object>();
|
||||
foreach (ColumnInfo info in row.schema.columnInfo)
|
||||
@@ -211,4 +203,4 @@ namespace BCG
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,8 @@
|
||||
using System;
|
||||
using bts;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
@@ -40,12 +44,12 @@ namespace BCG
|
||||
private IComparer<IBlueRow> myRightCompare;
|
||||
protected FuzzyRowComparer myFuzzyCompare;
|
||||
protected TimeSpan myTimeWindow;
|
||||
protected IBlueTableSchema myLeftSchema;
|
||||
protected IBlueTableSchema myRightSchema;
|
||||
protected IBlueTableSchema myOutputSchema;
|
||||
protected IBlueTableSchema? myLeftSchema;
|
||||
protected IBlueTableSchema? myRightSchema;
|
||||
protected IBlueTableSchema? myOutputSchema;
|
||||
private int myID = 0;
|
||||
|
||||
protected TimeWindowTable myStateRows;
|
||||
protected TimeWindowTable? myStateRows;
|
||||
|
||||
private DateTime myNextCull;
|
||||
const long MinCullIntervalMS = 1000;
|
||||
@@ -65,7 +69,7 @@ namespace BCG
|
||||
this.myFuzzyCompare = fuzzyCompare;
|
||||
}
|
||||
|
||||
public virtual IBlueTableSchema OutputSchema
|
||||
public virtual IBlueTableSchema? OutputSchema
|
||||
{
|
||||
get
|
||||
{
|
||||
@@ -150,7 +154,7 @@ namespace BCG
|
||||
int count;
|
||||
for (count = 0; count < list.Count; count++)
|
||||
{
|
||||
if ((DateTime)list[count][timeColumn] > expiry)
|
||||
if (list[count].GetValue<DateTime>(timeColumn) > expiry)
|
||||
{
|
||||
// Stop when we find the first row that we can keep. Since this is sorted all others are also kept.
|
||||
break;
|
||||
@@ -172,14 +176,17 @@ namespace BCG
|
||||
this.myFitness = fitness;
|
||||
}
|
||||
public IBlueRow myRightRow { get; private set; }
|
||||
public IBlueRow myLeftRow { get; set; }
|
||||
public IBlueRow? myLeftRow { get; set; }
|
||||
public double myFitness { get; private set; }
|
||||
public IBlueRow myMergedRow { get; set; }
|
||||
public IBlueRow? myMergedRow { get; set; }
|
||||
|
||||
#region IComparable<RowMatch> Members
|
||||
|
||||
public int CompareTo(RowMatch other)
|
||||
public int CompareTo(RowMatch? other)
|
||||
{
|
||||
if (other is null) {
|
||||
return 1;
|
||||
}
|
||||
return myFitness.CompareTo(other.myFitness);
|
||||
}
|
||||
|
||||
@@ -262,6 +269,10 @@ namespace BCG
|
||||
|
||||
private void HandleCorrection(List<RowMatch> corrections, BlueTable adds, BlueTable dels)
|
||||
{
|
||||
if (corrections.Count == 0) {
|
||||
return;
|
||||
}
|
||||
Log.notnull(myStateRows);
|
||||
// We already emitted correction.MergedRow so we need to redact that and
|
||||
// then emit correction.left as an add since it is no longer matched
|
||||
foreach (RowMatch correction in corrections.Distinct())
|
||||
@@ -285,7 +296,7 @@ namespace BCG
|
||||
int r = 0;
|
||||
for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; )
|
||||
{
|
||||
if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn])
|
||||
if (leftExpired[l].GetValue<DateTime>(myLeftTimeColumn) <= rightExpired[r].GetValue<DateTime>(myRightTimeColumn))
|
||||
{
|
||||
// Advance Left
|
||||
IEnumerable<RowMatch> rm = rightMatches[l];
|
||||
@@ -332,7 +343,7 @@ namespace BCG
|
||||
|
||||
private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable<RowMatch> rightMatches)
|
||||
{
|
||||
IBlueRow merged = null;
|
||||
IBlueRow? merged = null;
|
||||
if (rightMatches != null)
|
||||
{
|
||||
foreach (RowMatch match in rightMatches)
|
||||
@@ -366,9 +377,9 @@ namespace BCG
|
||||
/// Override this function to provide a custom row merge.
|
||||
/// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting.
|
||||
/// </summary>
|
||||
protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right)
|
||||
protected virtual IBlueRow MergeRows(IBlueRow? left, IBlueRow? right)
|
||||
{
|
||||
Dictionary<string, object> newRow = new Dictionary<string, object>();
|
||||
Dictionary<string, object?> newRow = new();
|
||||
|
||||
if (left == null && right == null) throw new ArgumentException("oh come on.");
|
||||
|
||||
@@ -393,14 +404,14 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
newRow["id"] = this.myID++;
|
||||
newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn];
|
||||
newRow[NEW_TIME_COLUMN] = (left != null) ? left[myLeftTimeColumn] : right![myRightTimeColumn];
|
||||
|
||||
return new BlueRow(this.OutputSchema, newRow);
|
||||
}
|
||||
|
||||
private IEnumerable<RowMatch> FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
|
||||
private IEnumerable<RowMatch>? FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
|
||||
{
|
||||
List<RowMatch> matches = null;
|
||||
List<RowMatch>? matches = null;
|
||||
|
||||
// Search through all rows in list and compare with the fuzzy comparer
|
||||
for (int x = 0; x < list.Count; x++)
|
||||
@@ -421,21 +432,15 @@ namespace BCG
|
||||
matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch);
|
||||
|
||||
// Mark the best match for this particular right hand match
|
||||
RowMatch oldMatch;
|
||||
if (this.myBestMatch.TryGetValue(match, out oldMatch))
|
||||
{
|
||||
if (currentMatch.myFitness < oldMatch.myFitness)
|
||||
{
|
||||
if (myBestMatch.TryGetValue(match, out var oldMatch)) {
|
||||
if (currentMatch.myFitness < oldMatch.myFitness) {
|
||||
this.myBestMatch[match] = currentMatch;
|
||||
if (oldMatch.myMergedRow != null)
|
||||
{
|
||||
if (oldMatch.myMergedRow != null) {
|
||||
// We already emitted oldMatch.MergedRow so we need to fix it
|
||||
corrections.Add(oldMatch);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
this.myBestMatch[match] = currentMatch;
|
||||
}
|
||||
}
|
||||
|
||||
+9
-2
@@ -7,6 +7,8 @@ using System.Text.Json;
|
||||
using bts;
|
||||
using bts.SilverPine;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
namespace BCG {
|
||||
//[DataStreamUserRespresentation("AggregateStream")]
|
||||
public class NonIncGroupByStream : ViewStream {
|
||||
@@ -80,7 +82,7 @@ namespace BCG {
|
||||
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
|
||||
|
||||
private void _main_stream_onchange(object nv) {
|
||||
DataPacket dp = nv as DataPacket;
|
||||
var dp = ExceptionAssert.Argument.Is<DataPacket>(nv);
|
||||
///grouping together
|
||||
_timer.Restart();
|
||||
IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
|
||||
@@ -109,7 +111,12 @@ namespace BCG {
|
||||
}
|
||||
// TODO,
|
||||
if (total_state != null)
|
||||
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
||||
myFireEvent(new DataPacket(
|
||||
total_state,
|
||||
total_state,
|
||||
myLastValue == null
|
||||
? new BlueTable(total_state.schema)
|
||||
: ExceptionAssert.of_type<DataPacket>(myLastValue).newRows));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -11,7 +10,8 @@ namespace BCG
|
||||
/// </summary>
|
||||
internal class JoinOperandRowIndex
|
||||
{
|
||||
private IDictionary myJoinColDict = new Hashtable (), myIdDict = new Hashtable ();
|
||||
readonly Dictionary<object, object> myJoinColDict = new();
|
||||
readonly Dictionary<object, object> myIdDict = new();
|
||||
private int myJoinColOrdinal, myIdColOrdinal;
|
||||
|
||||
/// <summary>
|
||||
@@ -35,18 +35,18 @@ namespace BCG
|
||||
public void add (IBlueRow row)
|
||||
{
|
||||
RowList list;
|
||||
if (myJoinColDict[row[myJoinColOrdinal]] == null)
|
||||
myJoinColDict[row[myJoinColOrdinal]] = list = new RowList ();
|
||||
if (myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value] == null)
|
||||
myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value] = list = new RowList ();
|
||||
else
|
||||
list = (RowList)myJoinColDict[row[myJoinColOrdinal]];
|
||||
myIdDict[row[myIdColOrdinal]] = list.add (row);
|
||||
list = (RowList)myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value];
|
||||
myIdDict[row[myIdColOrdinal] ?? DBNull.Value] = list.add (row);
|
||||
}
|
||||
|
||||
public void remove (IBlueRow row)
|
||||
{
|
||||
RowNode node = (RowNode)myIdDict[row[myIdColOrdinal]];
|
||||
((RowList)myJoinColDict[row[myJoinColOrdinal]]).remove (node);
|
||||
myIdDict.Remove (row[myIdColOrdinal]);
|
||||
RowNode node = (RowNode)myIdDict[row[myIdColOrdinal] ?? DBNull.Value];
|
||||
((RowList)myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value]).remove (node);
|
||||
myIdDict.Remove (row[myIdColOrdinal] ?? DBNull.Value);
|
||||
}
|
||||
|
||||
public RowList getMatchingRows (object key)
|
||||
|
||||
@@ -1,18 +1,20 @@
|
||||
using System;
|
||||
using BCG.Utilities;
|
||||
using System.Collections;
|
||||
using bts;
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
internal class JoinRow : IBlueRow
|
||||
{
|
||||
private RowNode myLeftNode, myRightNode;
|
||||
protected IBlueRow myLeftRow, myRightRow;
|
||||
RowNode? myLeftNode;
|
||||
RowNode? myRightNode;
|
||||
protected IBlueRow myLeftRow;
|
||||
protected IBlueRow? myRightRow;
|
||||
protected int myId;
|
||||
protected JoinTableSchema mySchema;
|
||||
private object[] myValues;
|
||||
|
||||
private object?[] myValues;
|
||||
|
||||
#region Constructor(s)
|
||||
|
||||
/// <summary>
|
||||
@@ -21,8 +23,8 @@ namespace BCG
|
||||
/// <param name="leftRow">left row of the join</param>
|
||||
/// <param name="rightRow">right row of the join</param>
|
||||
/// <param name="schema">schema of the joined row</param>
|
||||
public JoinRow (IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
|
||||
: this ((int)leftRow["id"], leftRow, rightRow, schema) {}
|
||||
public JoinRow(IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
|
||||
: this(leftRow.Id(), leftRow, rightRow, schema) { }
|
||||
|
||||
/// <summary>
|
||||
/// Constructs a row assigning it the ID provioded as the first argument
|
||||
@@ -31,7 +33,7 @@ namespace BCG
|
||||
/// <param name="leftRow">left row of the join</param>
|
||||
/// <param name="rightRow">right row of the join</param>
|
||||
/// <param name="schema">schema of the joined row</param>
|
||||
public JoinRow (int id, IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
|
||||
public JoinRow (int id, IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema)
|
||||
{
|
||||
myId = id;
|
||||
myLeftRow = leftRow;
|
||||
@@ -48,13 +50,13 @@ namespace BCG
|
||||
|
||||
public bool isPartialJoin{get{return myRightRow == null;}}
|
||||
|
||||
public RowNode leftNode
|
||||
public RowNode? leftNode
|
||||
{
|
||||
get{return myLeftNode;}
|
||||
set{myLeftNode = value;}
|
||||
}
|
||||
|
||||
public RowNode rightNode
|
||||
public RowNode? rightNode
|
||||
{
|
||||
get{return myRightNode;}
|
||||
set{myRightNode = value;}
|
||||
@@ -85,7 +87,7 @@ namespace BCG
|
||||
/// The key can either be the string corrsponding to the column name or
|
||||
/// the order of the column
|
||||
/// </summary>
|
||||
public virtual object this [object key]
|
||||
public virtual object? this [object key]
|
||||
{
|
||||
get
|
||||
{
|
||||
@@ -96,7 +98,7 @@ namespace BCG
|
||||
set {throw new NotSupportedException ("The IBlueRow is read only");}
|
||||
}
|
||||
|
||||
public object this [int index]
|
||||
public object? this [int index]
|
||||
{
|
||||
get
|
||||
{
|
||||
@@ -119,7 +121,7 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public object this [string colName] {get{return this[mySchema[colName].colNo];}}
|
||||
public object? this [string colName] {get{return this[mySchema[colName].colNo];}}
|
||||
|
||||
public ICollection Keys{get{return mySchema.columnNames;}}
|
||||
|
||||
@@ -133,9 +135,11 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||
public void Add(object key, object? value) =>
|
||||
throw new NotSupportedException("The IBlueRow is readonly");
|
||||
|
||||
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||
public void Clear() =>
|
||||
throw new NotSupportedException ("The IBlueRow is readonly");
|
||||
|
||||
public bool Contains (object key){return mySchema[(string)key] != null;}
|
||||
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
using System;
|
||||
using System.Data;
|
||||
using System.Collections;
|
||||
using System.Text.RegularExpressions;
|
||||
using BCG.Utilities;
|
||||
using BCG.Utilities.General;
|
||||
using bts;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class JoinStream : ViewStream
|
||||
@@ -111,8 +110,15 @@ namespace BCG
|
||||
{
|
||||
if (myPs == null)
|
||||
{
|
||||
JoinNameResolver resolver = new JoinNameResolver ((JoinTableSchema)myIjs.schema, myIjs.getSchema (true), myLeftCm.columnExpression,
|
||||
myIjs.getSchema (false), myRightCm.columnExpression);
|
||||
ArgumentNullException.ThrowIfNull(myIjs);
|
||||
ArgumentNullException.ThrowIfNull(myLeftCm);
|
||||
ArgumentNullException.ThrowIfNull(myRightCm);
|
||||
var resolver = new JoinNameResolver(
|
||||
ExceptionAssert.of_type<JoinTableSchema>(myIjs.schema),
|
||||
myIjs.getSchema(true),
|
||||
myLeftCm.columnExpression,
|
||||
myIjs.getSchema(false),
|
||||
myRightCm.columnExpression);
|
||||
IDictionary map = resolver.getMap ();
|
||||
map["id"] = "id";
|
||||
myPs = new ProjectStream (myIjs, map);
|
||||
@@ -123,26 +129,27 @@ namespace BCG
|
||||
Log.assert(false, "This call back should not have been called");
|
||||
}
|
||||
else
|
||||
myFireEvent ((o as ICloneable).Clone());
|
||||
myFireEvent(ExceptionAssert.of_type<ICloneable>(o).Clone());
|
||||
}
|
||||
|
||||
private void myProjectChangeHandler (object o)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(myIjs);
|
||||
if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1)
|
||||
myIjs.onChange -= myInternalJoinChangeHandler;
|
||||
myFireEvent ((o as ICloneable).Clone());
|
||||
myFireEvent(ExceptionAssert.of_type<ICloneable>(o).Clone());
|
||||
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber ()
|
||||
{
|
||||
myIjs.onChange += myInternalJoinChangeHandler;
|
||||
ExceptionAssert.not_null(myIjs).onChange += myInternalJoinChangeHandler;
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber ()
|
||||
{
|
||||
// how does this relate to the -= in myProjectChangeHandler?
|
||||
myIjs.onChange -= myInternalJoinChangeHandler;
|
||||
ExceptionAssert.not_null(myIjs).onChange -= myInternalJoinChangeHandler;
|
||||
|
||||
// if there is a project after the ijs we need to
|
||||
// unsubscribe to it otherwise references will be kept and
|
||||
@@ -159,7 +166,9 @@ namespace BCG
|
||||
/// </summary>
|
||||
/// <param name="leftPacket"></param>
|
||||
/// <param name="rightPacket"></param>
|
||||
public void test (DataPacket dp, bool isLeft) {myIjs.test (dp, isLeft);}
|
||||
public void test (DataPacket dp, bool isLeft) =>
|
||||
ExceptionAssert.not_null(myIjs).test(dp, isLeft);
|
||||
|
||||
/***********************************************************************************************/
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.IO;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -19,20 +19,13 @@ namespace BCG
|
||||
myRightIndex = new JoinDeletionIndex (mySchema.getRenamedName ("id", false), false);
|
||||
}
|
||||
|
||||
public IBlueRow this [int id]
|
||||
{
|
||||
get
|
||||
{
|
||||
if (myIdIndex[id] == null)
|
||||
throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
|
||||
return (IBlueRow)myIdIndex[id];
|
||||
}
|
||||
}
|
||||
public IBlueRow this [int id] =>
|
||||
myIdIndex[id] as IBlueRow ?? throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
|
||||
|
||||
#region JoinTable members
|
||||
public virtual void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp)
|
||||
public virtual void addRow (IBlueRow leftRow, IBlueRow? rightRow, DataPacket dp)
|
||||
{
|
||||
JoinRow newRow = new JoinRow (myNewRowId, leftRow, rightRow, mySchema);
|
||||
var newRow = new JoinRow(myNewRowId, leftRow, rightRow, mySchema);
|
||||
myIdIndex[myNewRowId ++] = newRow;
|
||||
((BlueTable)(dp.addedRows)).addRow (newRow);
|
||||
myLeftIndex.add (newRow);
|
||||
@@ -48,17 +41,17 @@ namespace BCG
|
||||
thisIndex = myLeftIndex;
|
||||
otherIndex = myRightIndex;
|
||||
}
|
||||
RowList matchingNodes = thisIndex.getMatchingRows (deletedRow["id"]);
|
||||
RowList matchingNodes = thisIndex.getMatchingRows (deletedRow.Id());
|
||||
foreach (RowNode node in matchingNodes)
|
||||
{
|
||||
JoinRow row = (JoinRow)node.row;
|
||||
RowNode correspondingNode = isLeft ? row.rightNode : row.leftNode;
|
||||
var correspondingNode = isLeft ? row.rightNode : row.leftNode;
|
||||
if (correspondingNode != null)
|
||||
otherIndex.remove (correspondingNode); //in case of partial left joinss
|
||||
myIdIndex.Remove (row["id"]);
|
||||
myIdIndex.Remove (row.Id());
|
||||
((BlueTable)dp.deletedRows).addRow (row);
|
||||
}
|
||||
thisIndex.removeAll (deletedRow["id"]);
|
||||
thisIndex.removeAll(deletedRow.Id());
|
||||
}
|
||||
#endregion
|
||||
#region ICollection members
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
using bts;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
/// <summary>
|
||||
@@ -13,10 +14,11 @@ namespace BCG
|
||||
public class JoinTableSchema : BlueTableSchema
|
||||
{
|
||||
private string myLeftJc = "", myRightJc = "", myLeftId = "", myRightId = "";
|
||||
private IDictionary myLeftNameToAlias = new Hashtable ();
|
||||
private IDictionary myRightNameToAlias = new Hashtable ();
|
||||
private IDictionary myAliasToLeftName = new Hashtable(),
|
||||
myAliasToRightName = new Hashtable();
|
||||
readonly Dictionary<string, string> myLeftNameToAlias = new();
|
||||
readonly Dictionary<string, string> myRightNameToAlias = new();
|
||||
readonly Dictionary<string, string> myAliasToLeftName = new();
|
||||
readonly Dictionary<string, string> myAliasToRightName = new();
|
||||
|
||||
/// <summary>
|
||||
/// Constructs the schema of the Jointable with the schema of the left and right tables. Incase of
|
||||
/// a name being there in both the shemas it prefixes the name with "L." or "R." accordingly. Column "id"
|
||||
@@ -85,9 +87,9 @@ namespace BCG
|
||||
renamed = false;
|
||||
}
|
||||
myIndexByColName["id"] = new ColumnInfo("id", myIndexByColName.Count, typeof(int));
|
||||
myIndexByColOrder[myIndexByColOrder.Length - 1] = (ColumnInfo)myIndexByColName["id"];
|
||||
myAliasToLeftName = ProjectHelper.invertDic(myLeftNameToAlias);
|
||||
myAliasToRightName = ProjectHelper.invertDic(myRightNameToAlias);
|
||||
myIndexByColOrder[myIndexByColOrder.Length - 1] = ExceptionAssert.of_type<ColumnInfo>(myIndexByColName["id"]);
|
||||
myAliasToLeftName = myLeftNameToAlias.ToDictionary(kv => kv.Value, kv => kv.Key);
|
||||
myAliasToRightName = myRightNameToAlias.ToDictionary(kv => kv.Key, kv => kv.Value);
|
||||
}
|
||||
|
||||
public string leftJoinCol {get{return myLeftJc;}}
|
||||
@@ -97,8 +99,8 @@ namespace BCG
|
||||
public string getRenamedName (string origName, bool isLeft)
|
||||
{
|
||||
if (isLeft)
|
||||
return (string)myLeftNameToAlias[origName];
|
||||
return (string)myRightNameToAlias[origName];
|
||||
return myLeftNameToAlias[origName] as string ?? string.Empty;
|
||||
return myRightNameToAlias[origName] as string ?? string.Empty;
|
||||
}
|
||||
|
||||
public string getOriginalName(string rename, bool isLeft)
|
||||
|
||||
+32
-20
@@ -2,16 +2,15 @@
|
||||
using bts.collections;
|
||||
|
||||
using mbts;
|
||||
using mbts.utils;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
|
||||
namespace BCG {
|
||||
public class NonIncJoinBookkeeper : IDisposable {
|
||||
public Action on_new_data;
|
||||
public Action? on_new_data;
|
||||
private readonly NonIncJoinSpec _join_spec;
|
||||
private readonly int _join_position;
|
||||
private readonly int _start_index_in_output;
|
||||
@@ -20,16 +19,19 @@ namespace BCG {
|
||||
private readonly bool _is_tag_along;
|
||||
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
||||
|
||||
private IBlueTableSchema _schema = null;
|
||||
private IBlueTableSchema? _schema;
|
||||
private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
|
||||
private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
|
||||
private IBlueRow _solitary_row = null;
|
||||
private IBlueRow? _solitary_row;
|
||||
|
||||
private int[] _join_col_indices = null,
|
||||
_data_col_indices = null;
|
||||
int[]? _join_col_indices;
|
||||
int[]? _data_col_indices;
|
||||
|
||||
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
|
||||
_join_spec = the_join_spec;
|
||||
public NonIncJoinBookkeeper(NonIncJoinSpec join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
|
||||
if (join_spec.join_col_names is null) {
|
||||
throw new ArgumentException("join_col_names is null", nameof(join_spec));
|
||||
}
|
||||
_join_spec = join_spec;
|
||||
_join_position = join_position;
|
||||
_start_index_in_output = start_index_in_output;
|
||||
_row_hasher = row_hasher;
|
||||
@@ -41,7 +43,7 @@ namespace BCG {
|
||||
private void _fill_missing_data(DataPacket dp) {
|
||||
if (_filled_missing_data)
|
||||
return;
|
||||
Func<string[], bool> null_or_empty = (str_arr) => {
|
||||
bool null_or_empty(string[]? str_arr) {
|
||||
if (str_arr == null || str_arr.Length == 0)
|
||||
return true;
|
||||
foreach (string s in str_arr) {
|
||||
@@ -50,13 +52,16 @@ namespace BCG {
|
||||
}
|
||||
}
|
||||
return true;
|
||||
};
|
||||
Func<string[], string[]> remove_bad_cols = (col_names) => {
|
||||
}
|
||||
string[] remove_bad_cols(string[]? col_names) {
|
||||
if (col_names is null) {
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
|
||||
List<string> cols_list = new List<string>(col_names);
|
||||
cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
|
||||
return cols_list.ToArray();
|
||||
};
|
||||
}
|
||||
if (null_or_empty(_join_spec.data_col_names)) {
|
||||
_join_spec.data_col_names = dp.schema.columnNames;
|
||||
}
|
||||
@@ -70,7 +75,7 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public void init() {
|
||||
_join_spec.stream.onChange += _on_stream_cb;
|
||||
ExceptionAssert.not_null(_join_spec.stream).onChange += _on_stream_cb;
|
||||
}
|
||||
|
||||
public void _on_stream_cb(object dp_obj) {
|
||||
@@ -111,14 +116,19 @@ namespace BCG {
|
||||
public bool is_ready { get { return _schema != null; } }
|
||||
|
||||
public void constribute_cols_to_schema(List<ColumnInfo> cis) {
|
||||
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
|
||||
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
|
||||
ArgumentNullException.ThrowIfNull(_join_spec.data_col_names);
|
||||
if (_join_spec.data_col_names.Length > 0) {
|
||||
ArgumentNullException.ThrowIfNull(_join_spec.projected_col_names);
|
||||
ArgumentNullException.ThrowIfNull(_schema);
|
||||
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
|
||||
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
|
||||
}
|
||||
}
|
||||
|
||||
public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
|
||||
|
||||
public void copy_master_row_to_output_row(int row_index, object[] output_row) {
|
||||
IBlueRow this_row = _row_list[row_index].second;
|
||||
IBlueRow? this_row = _row_list[row_index].second;
|
||||
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
||||
}
|
||||
|
||||
@@ -129,8 +139,7 @@ namespace BCG {
|
||||
else
|
||||
copy_dbnull_to_output_row(output_row);
|
||||
} else {
|
||||
IBlueRow this_row = null;
|
||||
if (_row_map.TryGetValue(row_hash, out this_row))
|
||||
if (_row_map.TryGetValue(row_hash, out var this_row))
|
||||
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
||||
else
|
||||
copy_dbnull_to_output_row(output_row);
|
||||
@@ -138,13 +147,16 @@ namespace BCG {
|
||||
}
|
||||
|
||||
private void copy_dbnull_to_output_row(object[] output_row) {
|
||||
ArgumentNullException.ThrowIfNull(_data_col_indices);
|
||||
for (int i = 0; i < _data_col_indices.Length; i++)
|
||||
output_row[i + _start_index_in_output] = DBNull.Value;
|
||||
}
|
||||
|
||||
#region IDisposable
|
||||
|
||||
public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
|
||||
public void Dispose() {
|
||||
ExceptionAssert.not_null(_join_spec.stream).onChange -= _on_stream_cb;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
@@ -1,20 +1,18 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using mbts;
|
||||
using mbts.utils;
|
||||
|
||||
namespace BCG {
|
||||
// this a collection of data, theres no joining logic in here
|
||||
public class NonIncJoinSpec {
|
||||
public ViewStream stream;
|
||||
public ViewStream? stream;
|
||||
// join col names are the names on which the streams are joined (for equality),
|
||||
// data col names are the names in the original stream
|
||||
// projected col names are their new names, ie if its x:l_x, then x is in the
|
||||
// data_col_names and l_x in projected cols
|
||||
public string[] join_col_names, data_col_names, projected_col_names;
|
||||
public string[]? join_col_names, data_col_names, projected_col_names;
|
||||
|
||||
public NonIncJoinSpec() { }
|
||||
|
||||
@@ -31,8 +29,8 @@ namespace BCG {
|
||||
public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
|
||||
List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
|
||||
foreach (Triplet<ViewStream, string, string> r in join_specs) {
|
||||
string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
|
||||
string[] visible_cols = r.third.Split(' ');
|
||||
string[] keys = (r.second ?? string.Empty).Split(' ');
|
||||
string[] visible_cols = ExceptionAssert.not_null(r.third).Split(' ');
|
||||
List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
|
||||
foreach (string s in visible_cols) {
|
||||
string[] toks = s.Split(':');
|
||||
@@ -44,22 +42,15 @@ namespace BCG {
|
||||
mapped_col_names.Add(toks[1]);
|
||||
}
|
||||
}
|
||||
rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
|
||||
rv.Add(new NonIncJoinSpec(ExceptionAssert.not_null(r.first), keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
|
||||
}
|
||||
return rv.ToArray();
|
||||
}
|
||||
|
||||
public override string ToString() {
|
||||
Func<string[], string> arr_txt = (a) => {
|
||||
string rv = "";
|
||||
foreach (string s in a) {
|
||||
rv += "{s} ";
|
||||
}
|
||||
rv = rv.Trim();
|
||||
return rv;
|
||||
};
|
||||
string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
|
||||
return rv;
|
||||
}
|
||||
public override string ToString() =>
|
||||
$"stream: {stream} "
|
||||
+ $"join_col_names: { string.Join(" ", join_col_names ?? Array.Empty<string>())} "
|
||||
+ $"data_col_names: { string.Join(" ", data_col_names ?? Array.Empty<string>())} "
|
||||
+ $"projected_col_names: { string.Join(" ", projected_col_names ?? Array.Empty<string>())}";
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,14 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
using bts;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using mbts;
|
||||
using mbts.utils;
|
||||
|
||||
using Spark.Core.Collections;
|
||||
|
||||
namespace BCG
|
||||
@@ -178,10 +185,11 @@ namespace BCG
|
||||
public class SuperJoinRow : IBlueRow
|
||||
{
|
||||
private JoinProjectHelper myHelper;
|
||||
private IBlueRow myLeftRow, myRightRow;
|
||||
private IBlueRow myLeftRow;
|
||||
private IBlueRow? myRightRow;
|
||||
int myID;
|
||||
|
||||
public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow rightRow, JoinProjectHelper joinProjectHelper)
|
||||
public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow? rightRow, JoinProjectHelper joinProjectHelper)
|
||||
{
|
||||
myID = id;
|
||||
myLeftRow = leftRow;
|
||||
@@ -189,21 +197,21 @@ namespace BCG
|
||||
myHelper = joinProjectHelper;
|
||||
}
|
||||
|
||||
public int leftID { get { return (int)myLeftRow["id"]; } }
|
||||
public int leftID => myLeftRow.Id();
|
||||
|
||||
public int rightID { get { return (int)myRightRow["id"]; } }
|
||||
public int rightID => ExceptionAssert.not_null(myRightRow).Id();
|
||||
|
||||
public IBlueRow leftRow { get { return myLeftRow; } }
|
||||
|
||||
public IBlueRow rightRow { get { return myRightRow; } }
|
||||
public IBlueRow? rightRow => myRightRow;
|
||||
|
||||
#region IBlueRow Members
|
||||
|
||||
public IBlueTableSchema schema { get { return myHelper.schema; } }
|
||||
|
||||
public object this[int index] { get { return this[schema[index].colName]; } }
|
||||
public object? this[int index] { get { return this[schema[index].colName]; } }
|
||||
|
||||
public object this[string colName]
|
||||
public object? this[string colName]
|
||||
{
|
||||
get
|
||||
{
|
||||
@@ -211,8 +219,7 @@ namespace BCG
|
||||
return myID;
|
||||
else
|
||||
{
|
||||
bool isLeft;
|
||||
string col = myHelper.getOriginalCol(colName, out isLeft);
|
||||
string col = myHelper.getOriginalCol(colName, out bool isLeft);
|
||||
return isLeft ? myLeftRow[col] : (myRightRow == null ? (object)DBNull.Value : myRightRow[col]);
|
||||
}
|
||||
}
|
||||
@@ -229,7 +236,7 @@ namespace BCG
|
||||
|
||||
public bool Contains(object key)
|
||||
{
|
||||
return myLeftRow.Contains(key) || myRightRow.Contains(key);
|
||||
return myLeftRow.Contains(key) || (myRightRow?.Contains(key) ?? false);
|
||||
}
|
||||
|
||||
public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
|
||||
@@ -246,7 +253,7 @@ namespace BCG
|
||||
{
|
||||
get
|
||||
{
|
||||
List<object> rv = new List<object>();
|
||||
var rv = new List<object?>();
|
||||
foreach (string col in Keys)
|
||||
rv.Add(this[col]);
|
||||
return rv;
|
||||
@@ -291,9 +298,10 @@ namespace BCG
|
||||
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
|
||||
private bool myManageUnmappedLeftRows = false;
|
||||
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
|
||||
protected IBlueTableSchema myLeftSchema, myRightSchema;
|
||||
protected IBlueTableSchema? myLeftSchema;
|
||||
protected IBlueTableSchema? myRightSchema;
|
||||
protected JoinProjectHelper myJoinProjectHelper;
|
||||
protected BlueTable myState;
|
||||
protected BlueTable? myState;
|
||||
protected int myID;
|
||||
protected ViewStream myLeftStream, myRightStream;
|
||||
protected string myLeftKeyCol, myRightKeyCol;
|
||||
@@ -346,25 +354,25 @@ namespace BCG
|
||||
|
||||
private void myHandleLeftStreamChanges(object newValue)
|
||||
{
|
||||
myHandleStreamChanges(newValue as DataPacket, true);
|
||||
myHandleStreamChanges(ExceptionAssert.Argument.Is<DataPacket>(newValue), true);
|
||||
}
|
||||
|
||||
private void myHandleRightStreamChanges(object newValue)
|
||||
{
|
||||
myHandleStreamChanges(newValue as DataPacket, false);
|
||||
myHandleStreamChanges(ExceptionAssert.Argument.Is<DataPacket>(newValue), false);
|
||||
}
|
||||
|
||||
private void myHandleStreamChanges(DataPacket dp, bool isLeft)
|
||||
{
|
||||
myInitSchema(dp.schema, isLeft);
|
||||
BlueTable dels = null, adds = null;
|
||||
BlueTable? dels = null, adds = null;
|
||||
if (myState != null)
|
||||
{
|
||||
dels = new BlueTable(myState.schema);
|
||||
adds = new BlueTable(myState.schema);
|
||||
}
|
||||
myDeleteRows(dp.deletedRows, isLeft, myState, dels);
|
||||
myAddRows(dp.addedRows, isLeft, myState, adds);
|
||||
myDeleteRows(dp.deletedRows, isLeft, ExceptionAssert.not_null(myState), ExceptionAssert.not_null(dels));
|
||||
myAddRows(dp.addedRows, isLeft, myState, ExceptionAssert.not_null(adds));
|
||||
if (myState != null)
|
||||
{
|
||||
myFireEvent(new DataPacket(myState, adds, dels));
|
||||
@@ -410,8 +418,8 @@ namespace BCG
|
||||
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
|
||||
foreach (IBlueRow r in addedRows)
|
||||
{
|
||||
object currentKey = r[thisKeyCol];
|
||||
int currentID = (int)r["id"];
|
||||
var currentKey = r[thisKeyCol] ?? DBNull.Value;
|
||||
int currentID = r.Id();
|
||||
///add it to 1.
|
||||
if (!thisKeyToRows.ContainsKey(currentKey))
|
||||
thisKeyToRows.Add(currentKey, new Dictionary<int, IBlueRow>());
|
||||
@@ -422,14 +430,14 @@ namespace BCG
|
||||
ICollection<IBlueRow> matchingRows = otherSideKeyToRows[currentKey].Values;
|
||||
foreach (IBlueRow mr in matchingRows)
|
||||
{
|
||||
int mrID = (int)mr["id"];
|
||||
int mrID = mr.Id();
|
||||
SuperJoinRow rr = new SuperJoinRow(myID++, isLeft ? r : mr, isLeft ? mr : r, myJoinProjectHelper);
|
||||
state.addRow(rr);
|
||||
addedRowsOut.addRow(rr);
|
||||
///add to 3.
|
||||
if (!thisIDToResultRows.ContainsKey(currentID))
|
||||
thisIDToResultRows.Add(currentID, new Dictionary<int, SuperJoinRow>());
|
||||
thisIDToResultRows[currentID].Add((int)rr["id"], rr);
|
||||
thisIDToResultRows[currentID].Add(rr.Id(), rr);
|
||||
///add to 4.
|
||||
if (!otherIDToResultRows.ContainsKey(mrID))
|
||||
{
|
||||
@@ -437,7 +445,7 @@ namespace BCG
|
||||
if (myManageUnmappedLeftRows && !isLeft)
|
||||
myTakeCareOfUnmatachedLeftRows(mr, false);
|
||||
}
|
||||
otherIDToResultRows[mrID].Add((int)rr["id"], rr);
|
||||
otherIDToResultRows[mrID].Add(rr.Id(), rr);
|
||||
}
|
||||
}
|
||||
else
|
||||
@@ -458,22 +466,21 @@ namespace BCG
|
||||
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
|
||||
foreach (IBlueRow r in deletedRows)
|
||||
{
|
||||
int id = (int)r["id"];
|
||||
object key = r[thisKeyCol];
|
||||
int id = r.Id();
|
||||
object key = ExceptionAssert.not_null(r[thisKeyCol]);
|
||||
///delete it from 1.
|
||||
thisKeyToRows[key].Remove(id);
|
||||
if (thisKeyToRows[key].Count == 0)
|
||||
thisKeyToRows.Remove(key);
|
||||
///form return rows and delete from 3.
|
||||
if (thisIDToResultRows.ContainsKey(id))
|
||||
if (thisIDToResultRows.TryGetValue(id, out IDictionary<int, SuperJoinRow>? removedResultRows))
|
||||
{
|
||||
ICollection removedResultRows = thisIDToResultRows[id].Values as ICollection;
|
||||
thisIDToResultRows.Remove(id);
|
||||
foreach (SuperJoinRow rr in removedResultRows)
|
||||
foreach (var rr in removedResultRows.Values)
|
||||
{
|
||||
///delete from 4.
|
||||
int otherId = isLeft ? rr.rightID : rr.leftID;
|
||||
otherIDToResultRows[otherId].Remove((int)rr["id"]);
|
||||
otherIDToResultRows[otherId].Remove(rr.Id());
|
||||
if (otherIDToResultRows[otherId].Count == 0)
|
||||
otherIDToResultRows.Remove(otherId);
|
||||
delRowsOut.addRow(rr);
|
||||
@@ -493,15 +500,11 @@ namespace BCG
|
||||
|
||||
private void myTakeCareOfUnmatachedLeftRows(IBlueRow leftRow, bool add)
|
||||
{
|
||||
int lid = (int)leftRow["id"];
|
||||
if (add)
|
||||
int lid = leftRow.Id();
|
||||
if (add) {
|
||||
myLeftUnmatchedRowsAdds.Add(lid, leftRow);
|
||||
else
|
||||
{
|
||||
if (myLeftUnmatchedRowsAdds.ContainsKey(lid))
|
||||
myLeftUnmatchedRowsAdds.Remove(lid);
|
||||
else
|
||||
myLeftUnmatchedRowsDels.Add((int)leftRow["id"], leftRow);
|
||||
} else if (!myLeftUnmatchedRowsAdds.Remove(lid)) {
|
||||
myLeftUnmatchedRowsDels.Add(lid, leftRow);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -522,44 +525,48 @@ namespace BCG
|
||||
|
||||
protected override void myFireEvent(object o)
|
||||
{
|
||||
myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, o as DataPacket);
|
||||
var dp = ExceptionAssert.Argument.Is<DataPacket>(o);
|
||||
myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, dp);
|
||||
}
|
||||
|
||||
private void myHandleJoinResults(Dictionary<int, IBlueRow> unmatchedLeftRowsDels,
|
||||
Dictionary<int, IBlueRow> unmatchedLeftRowAdds, DataPacket resultPacket)
|
||||
{
|
||||
BlueTable dels = resultPacket.deletedRows as BlueTable,
|
||||
adds = resultPacket.addedRows as BlueTable;
|
||||
|
||||
foreach (IBlueRow lr in unmatchedLeftRowsDels.Values)
|
||||
{
|
||||
///an lr will be here if its actually deleted or
|
||||
///a right row is added, so now it has a partner
|
||||
///and is no longer unmatched.
|
||||
///in both cases the half row that this lr
|
||||
///created needs to be removed from this
|
||||
///class's state and added to dels
|
||||
///it means a right is added
|
||||
int lid = (int)lr["id"];
|
||||
SuperJoinRow rr = myLeftIDToHalfRows[lid];
|
||||
myLeftIDToHalfRows.Remove(lid);
|
||||
myState.deleteRow(rr);
|
||||
dels.addRow(rr);
|
||||
var dels = ExceptionAssert.of_type<BlueTable>(resultPacket.deletedRows);
|
||||
var adds = ExceptionAssert.of_type<BlueTable>(resultPacket.addedRows);
|
||||
if (unmatchedLeftRowsDels.Count > 0) {
|
||||
Log.notnull(myState);
|
||||
foreach (IBlueRow lr in unmatchedLeftRowsDels.Values) {
|
||||
///an lr will be here if its actually deleted or
|
||||
///a right row is added, so now it has a partner
|
||||
///and is no longer unmatched.
|
||||
///in both cases the half row that this lr
|
||||
///created needs to be removed from this
|
||||
///class's state and added to dels
|
||||
///it means a right is added
|
||||
int lid = lr.Id();
|
||||
SuperJoinRow rr = myLeftIDToHalfRows[lid];
|
||||
myLeftIDToHalfRows.Remove(lid);
|
||||
myState.deleteRow(rr);
|
||||
dels.addRow(rr);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (IBlueRow lr in unmatchedLeftRowAdds.Values)
|
||||
{
|
||||
///an lr will be here if its added with no
|
||||
///matching right, or a right matching row
|
||||
///is gone so it is unmatched now.
|
||||
///in both cases a half row needs to be
|
||||
///added to this class's state and added
|
||||
///to adds
|
||||
int lid = (int)lr["id"];
|
||||
SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
|
||||
myLeftIDToHalfRows.Add(lid, rr);
|
||||
myState.addRow(rr);
|
||||
adds.addRow(rr);
|
||||
if (unmatchedLeftRowAdds.Count > 0) {
|
||||
Log.notnull(myState);
|
||||
foreach (IBlueRow lr in unmatchedLeftRowAdds.Values) {
|
||||
///an lr will be here if its added with no
|
||||
///matching right, or a right matching row
|
||||
///is gone so it is unmatched now.
|
||||
///in both cases a half row needs to be
|
||||
///added to this class's state and added
|
||||
///to adds
|
||||
int lid = lr.Id();
|
||||
var rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
|
||||
myLeftIDToHalfRows.Add(lid, rr);
|
||||
myState.addRow(rr);
|
||||
adds.addRow(rr);
|
||||
}
|
||||
}
|
||||
base.myFireEvent(new DataPacket(myState, adds, dels));
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
using bts;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -11,9 +14,10 @@ namespace BCG
|
||||
[DataStreamUserRespresentation (false)]
|
||||
public class LeftInternalJoinStream : InternalJoinStream
|
||||
{
|
||||
protected DataPacket myLeftDataPacket, myRightDataPacket;
|
||||
protected DataPacket? myLeftDataPacket, myRightDataPacket;
|
||||
//Container to contain all the left rows that are orphaned during a deletion operation
|
||||
private IDictionary myLeftOrphanedRows;
|
||||
IDictionary? myLeftOrphanedRows;
|
||||
|
||||
public LeftInternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight) :
|
||||
base (vsLeft, jcLeft, vsRight, jcRight) {}
|
||||
|
||||
@@ -62,10 +66,10 @@ namespace BCG
|
||||
private void myTestAndFire (bool isLeft)
|
||||
{
|
||||
JoinOperand jo = isLeft ? myLeftOperand : myRightOperand;
|
||||
DataPacket dp = isLeft ? myLeftDataPacket : myRightDataPacket;
|
||||
var dp = isLeft ? myLeftDataPacket : myRightDataPacket;
|
||||
//to test whether both have received call backs
|
||||
if (myLeftOperand.hasReceivedData)
|
||||
jo.changeHandler (dp);
|
||||
jo.changeHandler(Log.notnull(dp));
|
||||
else if (myLeftDataPacket != null && myRightDataPacket != null)
|
||||
{
|
||||
myRightOperand.changeHandler (myRightDataPacket);
|
||||
@@ -82,9 +86,11 @@ namespace BCG
|
||||
{
|
||||
foreach (IBlueRow row in rows)
|
||||
myAddRow (row, isLeft);
|
||||
if (!isLeft)
|
||||
if (!isLeft) {
|
||||
Log.notnull(myLeftOrphanedRows);
|
||||
foreach (RowNode node in myLeftOrphanedRows.Values)
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow (node.row, null, myLastFiredPacket);
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow(node.row, null, myLastFiredPacket);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -104,7 +110,7 @@ namespace BCG
|
||||
if (isLeft)
|
||||
{
|
||||
myLeftOperand.addRow (row);
|
||||
RowList matchedRows = myRightOperand.getMatchingRows (row[myLeftJoinColOrdinal]);
|
||||
RowList matchedRows = myRightOperand.getMatchingRows (row[myLeftJoinColOrdinal] ?? DBNull.Value);
|
||||
if (matchedRows.Count == 0)
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
|
||||
else
|
||||
@@ -113,10 +119,10 @@ namespace BCG
|
||||
}
|
||||
else
|
||||
{
|
||||
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]);
|
||||
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
|
||||
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value);
|
||||
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count;
|
||||
myRightOperand.addRow (row);
|
||||
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
|
||||
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count;
|
||||
if (matchedRows.Count != 0){
|
||||
foreach (RowNode matchedNode in matchedRows)
|
||||
{
|
||||
@@ -125,10 +131,11 @@ namespace BCG
|
||||
//the jointable as a new row will replace it
|
||||
if (rightMatchingRowCount == 0 && newRightMatchingRowCount > 0)
|
||||
{
|
||||
if (myLeftOrphanedRows[matchedNode.row["id"]] == null)
|
||||
Log.notnull(myLeftOrphanedRows);
|
||||
if (myLeftOrphanedRows[matchedNode.row.Id()] == null)
|
||||
((JoinTable)myLastFiredPacket.newRows).deleteRow (true, matchedNode.row, myLastFiredPacket);
|
||||
else
|
||||
myLeftOrphanedRows.Remove (matchedNode.row["id"]);
|
||||
myLeftOrphanedRows.Remove (matchedNode.row.Id());
|
||||
}
|
||||
//they were orphaned by a previous transaction and hence the orphans need to be
|
||||
//removed from the Jointable and the complete row added
|
||||
@@ -148,21 +155,23 @@ namespace BCG
|
||||
}
|
||||
else
|
||||
{
|
||||
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]);
|
||||
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
|
||||
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value);
|
||||
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count;
|
||||
myRightOperand.deleteRow (row);
|
||||
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
|
||||
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count;
|
||||
if (matchedRows.Count != 0)
|
||||
{
|
||||
if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0)
|
||||
if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0) {
|
||||
//ie all the matchedRows were joined with 'this' right row (method arg), and hence they need to
|
||||
//be handled specially, ie joined with nulls, if there are no right rows with this
|
||||
//joincolumn value being inserted.
|
||||
Log.notnull(myLeftOrphanedRows);
|
||||
foreach (RowNode matchedNode in matchedRows)
|
||||
{
|
||||
// BlueEventLog.logAssert (myLeftOrphanedRows[matchedNode.row["id"]] == null);
|
||||
myLeftOrphanedRows[matchedNode.row["id"]] = matchedNode;
|
||||
myLeftOrphanedRows[matchedNode.row.Id()] = matchedNode;
|
||||
}
|
||||
}
|
||||
((JoinTable)myLastFiredPacket.newRows).deleteRow (false, row, myLastFiredPacket);
|
||||
}
|
||||
}
|
||||
@@ -173,9 +182,11 @@ namespace BCG
|
||||
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
||||
myLastFiredPacket.newRows = new JoinTable (mySchema);
|
||||
myLastFiredPacket.addedRows = new BlueTable (mySchema);
|
||||
if (!isLeft)
|
||||
if (!isLeft) {
|
||||
Log.notnull(myLeftDataPacket);
|
||||
foreach (IBlueRow row in myLeftDataPacket.newRows)
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow(row, null, myLastFiredPacket);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ namespace BCG
|
||||
/// <summary>
|
||||
/// Node containing the referencedRow
|
||||
/// </summary>
|
||||
IBlueRow referencedRow {get;}
|
||||
IBlueRow? referencedRow {get;}
|
||||
|
||||
/// <summary>
|
||||
/// removes all referenced or referencing nodes (indicated by the bool), p.s. theres only one referenced row
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
using System;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
/// <summary>
|
||||
@@ -15,12 +13,12 @@ namespace BCG
|
||||
/// </summary>
|
||||
/// <param name="id"></param>
|
||||
/// <returns></returns>
|
||||
IBlueRow getRow (int id);
|
||||
IBlueRow? getRow (int id);
|
||||
/// <summary>
|
||||
/// Gets all the rows in the node
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
IBlueRow[] getRows ();
|
||||
IBlueRow?[] getRows ();
|
||||
/// <summary>
|
||||
/// Replaces the row with the given id, with the row passed as the second arg
|
||||
/// </summary>
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
using BCG;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -56,18 +56,18 @@ namespace BCG
|
||||
/// The key can either be the string corrsponding to the column name or
|
||||
/// the order of the column
|
||||
/// </summary>
|
||||
public virtual object this [object key]
|
||||
public virtual object? this [object key]
|
||||
{
|
||||
get {return key is string ? this [key as string] : this [(int)key];}
|
||||
get => key is string ks ? this[ks] : this[(int)key];
|
||||
set {throw new NotSupportedException ("The IBlueRow is read only");}
|
||||
}
|
||||
|
||||
public virtual object this [int index]
|
||||
public virtual object? this [int index]
|
||||
{
|
||||
get
|
||||
{
|
||||
if (index == mySchema.columnNames.Length - 1)
|
||||
return myRingGRow["id"];
|
||||
return myRingGRow.Id();
|
||||
if (index < myRingGRow.Count)
|
||||
return myRingGRow[index];
|
||||
else if (index >= myRingGRow.Count && index < mySchema.columnNames.Length - 1)
|
||||
@@ -77,7 +77,8 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public object this [string colName] {get{return this[mySchema[colName].colNo];}}
|
||||
public object? this [string colName] =>
|
||||
this[mySchema[colName].colNo];
|
||||
|
||||
public ICollection Keys{get{return mySchema.columnNames;}}
|
||||
|
||||
|
||||
+18
-7
@@ -1,6 +1,7 @@
|
||||
using mbts.utils;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -70,7 +71,10 @@ namespace BCG
|
||||
}
|
||||
else
|
||||
{
|
||||
SimpleNode sn = n as SimpleNode;
|
||||
var sn = ExceptionAssert.Argument.Is<SimpleNode>(n);
|
||||
if (sn.prev is null) {
|
||||
throw new ArgumentException("prev is null", nameof(n));
|
||||
}
|
||||
sn.prev.next = sn.next;
|
||||
if (sn.next != null)
|
||||
sn.next.prev = sn.prev;
|
||||
@@ -81,7 +85,7 @@ namespace BCG
|
||||
|
||||
public int count {get {return myFirst.listCount;}}
|
||||
|
||||
public IBlueRow referencedRow {get {return myFirst.row;}}
|
||||
public IBlueRow? referencedRow => myFirst.row;
|
||||
|
||||
public void removeAll (bool isReD)
|
||||
{
|
||||
@@ -110,10 +114,17 @@ namespace BCG
|
||||
public class LightForeignJoinRowListEnumerator : IEnumerator
|
||||
{
|
||||
private ILightForeignJoinRowList myList;
|
||||
private SimpleNode myPtr, myFirst;
|
||||
public LightForeignJoinRowListEnumerator (ILightForeignJoinRowList list, SimpleNode first)
|
||||
{myList = list; myFirst = first;}
|
||||
public object Current {get{return myPtr.row;}}
|
||||
private SimpleNode? myPtr;
|
||||
private SimpleNode myFirst;
|
||||
|
||||
public LightForeignJoinRowListEnumerator(ILightForeignJoinRowList list, SimpleNode first) {
|
||||
myList = list;
|
||||
myFirst = first;
|
||||
}
|
||||
|
||||
public object? Current =>
|
||||
myPtr is null ? throw new InvalidOperationException("myPtr is null") : myPtr.row;
|
||||
|
||||
public bool MoveNext() {return (myPtr = myPtr == null ? myFirst : myPtr.next) != null;}
|
||||
public void Reset() {myPtr = null;}
|
||||
public ILightForeignJoinRowList list {get {return myList;}}
|
||||
|
||||
+4
-3
@@ -1,5 +1,6 @@
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using BCG.Utilities;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -13,12 +14,12 @@ namespace BCG
|
||||
: base (reDRow, rinGRow, schema)
|
||||
{}
|
||||
|
||||
public override object this [int index]
|
||||
public override object? this [int index]
|
||||
{
|
||||
get
|
||||
{
|
||||
if (index == mySchema.columnNames.Length - 1)
|
||||
return myRingGRow["id"];
|
||||
return myRingGRow.Id();
|
||||
if (index < myRingGRow.Count)
|
||||
return myRingGRow[index];
|
||||
else if (index >= myRingGRow.Count && index < mySchema.columnNames.Length - 1)
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
using System;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
/// <summary>
|
||||
@@ -9,15 +7,15 @@ namespace BCG
|
||||
/// </summary>
|
||||
public class SimpleNode : INode
|
||||
{
|
||||
public IBlueRow row;
|
||||
public SimpleNode next, prev;
|
||||
public IBlueRow? row;
|
||||
public SimpleNode? next;
|
||||
public SimpleNode? prev;
|
||||
|
||||
public SimpleNode (IBlueRow r)
|
||||
: this (r, null, null)
|
||||
{}
|
||||
|
||||
public SimpleNode (IBlueRow r, SimpleNode nx, SimpleNode pv)
|
||||
{
|
||||
public SimpleNode(IBlueRow r, SimpleNode? nx, SimpleNode? pv) {
|
||||
row = r;
|
||||
next = nx;
|
||||
prev = pv;
|
||||
@@ -25,13 +23,15 @@ namespace BCG
|
||||
|
||||
#region INode Members
|
||||
|
||||
public IBlueRow getRow (int id) {return row;}
|
||||
public IBlueRow? getRow (int id) =>
|
||||
row;
|
||||
|
||||
public IBlueRow[] getRows() {return new IBlueRow[] {row};}
|
||||
public IBlueRow?[] getRows() =>
|
||||
new[] { row };
|
||||
|
||||
public void replace (int id, IBlueRow nr) {row = nr;}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,8 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
using Amazon.Runtime.Internal.Util;
|
||||
|
||||
using BCG;
|
||||
using BCG.Utilities;
|
||||
|
||||
using bts;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG {
|
||||
// Prefixing with "old" to avoid name conflict with the BlueOrb LiteralStream
|
||||
[DataStreamUserRespresentation("oldLiteral")]
|
||||
@@ -44,13 +38,14 @@ namespace BCG {
|
||||
return new BlueRow(map);
|
||||
};
|
||||
string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
|
||||
BlueTable table = null;
|
||||
BlueTable? table = null;
|
||||
foreach (string json_string in row_arr) {
|
||||
IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
|
||||
get_row(json_string, table == null ? 0 : table.Count);
|
||||
table = table ?? new BlueTable(r.schema);
|
||||
table ??= new BlueTable(r.schema);
|
||||
table.addRow(r);
|
||||
}
|
||||
table ??= BlueTable.Empty(table_name);
|
||||
ViewStream str = new BlueTableStream(table);
|
||||
str = new SchemaGuesserStream(str);
|
||||
mySetRealSource(str);
|
||||
@@ -67,8 +62,11 @@ namespace BCG {
|
||||
string[] columnAndType = c.Split('/');
|
||||
if (columnAndType.Length == 1)
|
||||
ci[n] = new ColumnInfo(c, n, typeof(double));
|
||||
else
|
||||
ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
|
||||
else {
|
||||
var colType = Type.GetType("System." + columnAndType[1], true)
|
||||
?? throw new ArgumentException($"Failed to get column type at index {n}", nameof(schema));
|
||||
ci[n] = new ColumnInfo(columnAndType[0], n, colType);
|
||||
}
|
||||
n++;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
using bts;
|
||||
|
||||
using Bts.ColumnMapExpressions;
|
||||
using Bts.DataStream;
|
||||
|
||||
using mbts.config;
|
||||
using mbts.utils;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
// using BCG.Evaluator.ColumnMapExpression;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -26,7 +27,7 @@ namespace BCG
|
||||
/// <param name="isch">original(unprojected) schema</param>
|
||||
/// <param name="exp">expression defining the mapping</param>
|
||||
/// <param name="tableName">table name of the new schema</param>
|
||||
internal ProjectHelper (IBlueTableSchema isch, string exp, string tableName)
|
||||
internal ProjectHelper (IBlueTableSchema isch, string exp, string? tableName)
|
||||
: this (isch, myGetDictionary (isch, exp), tableName)
|
||||
{}
|
||||
|
||||
@@ -36,10 +37,10 @@ namespace BCG
|
||||
/// <param name="origToAlias">dictionary mapping unprojected names to their projected names, dropped cols should not be there</param>
|
||||
/// <param name="tableName">table name of the new schema</param>
|
||||
internal ProjectHelper (
|
||||
IBlueTableSchema isch, IDictionary<string, string> origToAlias, string tableName)
|
||||
IBlueTableSchema isch, IDictionary<string, string> origToAlias, string? tableName)
|
||||
{
|
||||
ArrayList colMapAl = new ArrayList (),
|
||||
colInfoAl = new ArrayList ();
|
||||
var colMapAl = new List<int>();
|
||||
var colInfoAl = new List<ColumnInfo>();
|
||||
relevantOrigCols = new string[origToAlias.Count-1];
|
||||
int c = 0;
|
||||
ArrayList iColNames = new ArrayList (isch.columnNames);
|
||||
@@ -49,19 +50,15 @@ namespace BCG
|
||||
if (origToAlias.TryGetValue(colName, out var colAlias))
|
||||
{
|
||||
ColumnInfo ci = isch[colName];
|
||||
colInfoAl.Add (new ColumnInfo (colAlias,
|
||||
colInfoAl.Count,
|
||||
ci.colType)
|
||||
);
|
||||
colMapAl.Add (ci.colNo);
|
||||
colInfoAl.Add(new ColumnInfo (colAlias, colInfoAl.Count, ci.colType));
|
||||
colMapAl.Add(ci.colNo);
|
||||
if (ci.colName != "id")
|
||||
relevantOrigCols[c++] = ci.colName;
|
||||
}
|
||||
}
|
||||
|
||||
schema = new BlueTableSchema (tableName == null ? isch.tableName : tableName,
|
||||
colInfoAl.ToArray (typeof(ColumnInfo)) as ColumnInfo[]);
|
||||
colMap = colMapAl.ToArray (typeof(int)) as int[];
|
||||
schema = new BlueTableSchema(tableName ?? isch.tableName, colInfoAl.ToArray());
|
||||
colMap = colMapAl.ToArray();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -75,16 +72,11 @@ namespace BCG
|
||||
{
|
||||
IDictionary<string, string> origToAlias = new Dictionary<string, string>();
|
||||
|
||||
ColumnMapExpression e;
|
||||
if (Configuration.get<DataStreamConfig>().UseStaticParser)
|
||||
{
|
||||
e = ColumnMapExpression.Parse(exp);
|
||||
}
|
||||
else
|
||||
{
|
||||
var e = Configuration.get<DataStreamConfig>().UseStaticParser
|
||||
? ColumnMapExpression.Parse(exp)
|
||||
// TODO (maybe): Actually use non-static parser (unclear what advantages it will have)
|
||||
e = ColumnMapExpression.Parse(exp);
|
||||
}
|
||||
: ColumnMapExpression.Parse(exp);
|
||||
Log.notnull(e);
|
||||
if (e.IsInclusive) {
|
||||
origToAlias = e.Columns.ToDictionary();
|
||||
origToAlias["id"] = "id";
|
||||
@@ -104,7 +96,7 @@ namespace BCG
|
||||
{
|
||||
IDictionary retVal = new Hashtable (dic.Count);
|
||||
foreach (DictionaryEntry de in dic)
|
||||
retVal.Add (de.Value, de.Key);
|
||||
retVal.Add(ExceptionAssert.not_null(de.Value), de.Key);
|
||||
return retVal;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
namespace BCG {
|
||||
using bts;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using Spark.Core.Collections;
|
||||
|
||||
using System.Collections;
|
||||
@@ -8,10 +12,11 @@ namespace BCG {
|
||||
[DataStreamUserRespresentation("oldproject")]
|
||||
public class ProjectStream : ViewStream
|
||||
{
|
||||
private ProjectHelper myHelper;
|
||||
private string myColExp, myTableName;
|
||||
private ViewStream myVs;
|
||||
private IDictionary<string, string> myDic;
|
||||
ProjectHelper? myHelper;
|
||||
string? myColExp;
|
||||
string? myTableName;
|
||||
ViewStream? myVs;
|
||||
IDictionary<string, string>? myDic;
|
||||
/// <summary>
|
||||
/// </summary>
|
||||
/// <param name="vs">stream to susbsribe to as the input stream</param>
|
||||
@@ -54,17 +59,17 @@ namespace BCG {
|
||||
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
myVs.onChange += myHandleChange;
|
||||
Log.notnull(myVs).onChange += myHandleChange;
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
myVs.onChange -= myHandleChange;
|
||||
Log.notnull(myVs).onChange -= myHandleChange;
|
||||
}
|
||||
|
||||
internal void myHandleChange (object newValue)
|
||||
{
|
||||
DataPacket dp = newValue as DataPacket;
|
||||
var dp = ExceptionAssert.Argument.Is<IDataPacket>(newValue);
|
||||
if (myHelper != null)
|
||||
{
|
||||
IBlueRowCollection dels, adds;
|
||||
@@ -79,8 +84,8 @@ namespace BCG {
|
||||
else
|
||||
{ //first time a callback is received
|
||||
myHelper = myDic == null ?
|
||||
new ProjectHelper (dp.schema, myColExp, myTableName) :
|
||||
new ProjectHelper (dp.schema, myDic, myTableName);
|
||||
new ProjectHelper (dp.Schema, myColExp ?? string.Empty, myTableName) :
|
||||
new ProjectHelper (dp.Schema, myDic, myTableName);
|
||||
|
||||
IBlueRowCollection temp;
|
||||
myFireEvent (new DataPacket (temp = new ProjectedTable (dp.newRows, myHelper),
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.IO;
|
||||
using BCG.Utilities;
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Data.Analysis;
|
||||
using Parquet;
|
||||
using Parquet.Data;
|
||||
using mbts.utils;
|
||||
|
||||
using Microsoft.Data.Analysis;
|
||||
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
|
||||
namespace BCG {
|
||||
[DataStreamUserRespresentation("CSVSaver")]
|
||||
@@ -40,7 +37,7 @@ namespace BCG {
|
||||
myStream.onChange -= myHandleChanges;
|
||||
|
||||
private void myHandleChanges(object newValue) {
|
||||
IBlueRowCollection totalState = (newValue as DataPacket).newRows;
|
||||
IBlueRowCollection totalState = ExceptionAssert.Argument.Is<DataPacket>(newValue).newRows;
|
||||
if (totalState.Count > 0 && myNWrites < myNMaxWrites) {
|
||||
myNWrites++;
|
||||
int i = 0;
|
||||
@@ -62,7 +59,10 @@ namespace BCG {
|
||||
IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
|
||||
BlueTable nRows = new BlueTable(myResultSchema);
|
||||
nRows.addRow(nr);
|
||||
myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
|
||||
BlueTable deletedRows = myLastValue is null
|
||||
? new(myResultSchema)
|
||||
: new(ExceptionAssert.of_type<DataPacket>(myLastValue).newRows);
|
||||
myFireEvent(new DataPacket(nRows, nRows, deletedRows));
|
||||
} else
|
||||
myFireEvent(newValue);
|
||||
}
|
||||
@@ -83,7 +83,7 @@ namespace BCG {
|
||||
if (col != "id") {
|
||||
string colText = String.Empty;
|
||||
if (r[col] != null && r[col] != DBNull.Value)
|
||||
colText = r[col].ToString();
|
||||
colText = r[col]?.ToString() ?? string.Empty;
|
||||
if (r.schema[col].colType == typeof(string))
|
||||
colText = "\"" + colText + "\"";
|
||||
rv += (i++ == 0 ? colText : "," + colText);
|
||||
@@ -136,7 +136,7 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public class CSVReaderStreamMapper : IStreamMapper {
|
||||
private IBlueTableSchema mySchema;
|
||||
IBlueTableSchema? mySchema;
|
||||
private readonly string[] myOutputSchemaStringArray;
|
||||
|
||||
public CSVReaderStreamMapper(string outputSchema) {
|
||||
@@ -162,6 +162,7 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public IBlueRow map(IBlueRow row) {
|
||||
ArgumentNullException.ThrowIfNull(mySchema);
|
||||
object[] vals = new object[row.schema.columnInfo.Length];
|
||||
row.Values.CopyTo(vals, 0);
|
||||
for (int i = 0; i < mySchema.columnInfo.Length; i++) {
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
using Bts.ColumnSortExpressions;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
@@ -39,7 +41,7 @@ public class SortRow : IComparable, IBlueRow {
|
||||
var otherVal = otherRow[col.Name];
|
||||
int rv = myIsNull(this[col.Name]) ?
|
||||
(myIsNull(otherVal) ? 0 : -1) :
|
||||
(myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal));
|
||||
(myIsNull(otherVal) ? 1 : ExceptionAssert.of_type<IComparable>(this[col.Name]).CompareTo(otherVal));
|
||||
rv *= col.IsAscending ? 1 : -1;
|
||||
return rv;
|
||||
}
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using BCG.Utilities.General;
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -49,8 +48,7 @@ namespace BCG
|
||||
#region IWorker
|
||||
public void doWork()
|
||||
{
|
||||
if ((myNextWrite <= DateTime.Now && myGotUpdate))
|
||||
{
|
||||
if (myNextWrite <= DateTime.Now && myGotUpdate) {
|
||||
myGotUpdate = false;
|
||||
myNextWrite = DateTime.Now.AddSeconds(myWriteFreqSecs);
|
||||
myWrite(myStream.rows);
|
||||
@@ -96,7 +94,7 @@ namespace BCG
|
||||
rowStr += DELIM;
|
||||
}
|
||||
|
||||
rowStr += row[columnName].ToString();
|
||||
rowStr += row[columnName]?.ToString() ?? string.Empty;
|
||||
}
|
||||
|
||||
writer.WriteLine(rowStr);
|
||||
@@ -139,7 +137,7 @@ namespace BCG
|
||||
private int myWriteFreqSecs;
|
||||
private DateTime myNextWrite;
|
||||
private bool myGotUpdate;
|
||||
private string mySchemaStr;
|
||||
private string? mySchemaStr;
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,8 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -11,7 +10,7 @@ namespace BCG
|
||||
{
|
||||
private ViewStream _src;
|
||||
private string _time_stamp_field;
|
||||
private IBlueTableSchema _schema;
|
||||
private IBlueTableSchema? _schema;
|
||||
private string _latency_field;
|
||||
|
||||
public LatencyStream(ViewStream input, string time_stamp_field, string latency_field)
|
||||
@@ -38,10 +37,10 @@ namespace BCG
|
||||
public IBlueRow map(IBlueRow row)
|
||||
{
|
||||
var ht = new Hashtable(row);
|
||||
if (row[_time_stamp_field].Equals(DBNull.Value))
|
||||
if (Equals(row[_time_stamp_field], DBNull.Value))
|
||||
ht[_latency_field] = DBNull.Value;
|
||||
else
|
||||
ht[_latency_field] = (DateTime.Now - (DateTime)row[_time_stamp_field]).TotalMilliseconds;
|
||||
ht[_latency_field] = (DateTime.Now - row.GetValue<DateTime>(_time_stamp_field)).TotalMilliseconds;
|
||||
return new BlueRow(_schema, ht);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
using System;
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -42,8 +43,8 @@ namespace BCG
|
||||
BlueTable result = new BlueTable(dp.schema);
|
||||
foreach (IBlueRow r in dp.newRows)
|
||||
{
|
||||
string key = r[myCol_Key].ToString();
|
||||
DateTime dt = (DateTime) r[myCol_Time];
|
||||
var key = r[myCol_Key]?.ToString() ?? string.Empty;
|
||||
var dt = r.GetValue<DateTime>(myCol_Time);
|
||||
double value = Convert.ToDouble(r[myCol_Value]);
|
||||
|
||||
object[] rowValues = new object[r.Count];
|
||||
@@ -68,9 +69,7 @@ namespace BCG
|
||||
private double myUpdateValue(string key, DateTime time, double value)
|
||||
{
|
||||
|
||||
SortedList<DateTime, double> seriesValues;
|
||||
if (!myKeyToTimeToValue.TryGetValue(key, out seriesValues))
|
||||
{
|
||||
if (!myKeyToTimeToValue.TryGetValue(key, out SortedList<DateTime, double>? seriesValues)) {
|
||||
seriesValues = new SortedList<DateTime, double>();
|
||||
myKeyToTimeToValue[key] = seriesValues;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
|
||||
@@ -12,7 +16,7 @@ namespace BCG
|
||||
private int myId = 0;
|
||||
private IDictionary myLeftIndex = new Hashtable (),
|
||||
myRightIndex = new Hashtable ();
|
||||
private IBlueTableSchema mySchema;
|
||||
private IBlueTableSchema? mySchema;
|
||||
|
||||
public AppendCollection () {}
|
||||
|
||||
@@ -24,15 +28,15 @@ namespace BCG
|
||||
IDictionary index = isLeft ? myLeftIndex : myRightIndex;
|
||||
foreach (IBlueRow r in dp.deletedRows)
|
||||
{
|
||||
IBlueRow dr = index[r["id"]] as IBlueRow;
|
||||
var dr = ExceptionAssert.of_type<IBlueRow>(index[r.Id()]);
|
||||
dRows.addRow (dr);
|
||||
index.Remove (r["id"]);
|
||||
index.Remove (r.Id());
|
||||
}
|
||||
foreach (IBlueRow r in dp.addedRows)
|
||||
{
|
||||
IBlueRow ar = new AppendRow (r, myId++, mySchema);
|
||||
aRows.addRow (ar);
|
||||
index.Add (r["id"], ar);
|
||||
index.Add(r.Id(), ar);
|
||||
}
|
||||
return new DataPacket (this, aRows, dRows);
|
||||
}
|
||||
@@ -43,7 +47,8 @@ namespace BCG
|
||||
|
||||
public bool containsID (int id) {throw new ApplicationException ("IBlueRow is readonly");}
|
||||
|
||||
public IBlueTableSchema schema {get {return mySchema;}}
|
||||
public IBlueTableSchema schema =>
|
||||
ExceptionAssert.not_null(mySchema);
|
||||
|
||||
#endregion
|
||||
|
||||
@@ -60,15 +65,14 @@ namespace BCG
|
||||
array.SetValue (r, i++);
|
||||
}
|
||||
|
||||
public object SyncRoot {get {return null;}}
|
||||
public object SyncRoot => this;
|
||||
|
||||
#endregion
|
||||
|
||||
#region IEnumerable Members
|
||||
|
||||
public System.Collections.IEnumerator GetEnumerator()
|
||||
{
|
||||
return new MyEnumerator (myLeftIndex, myRightIndex);
|
||||
public System.Collections.IEnumerator GetEnumerator() {
|
||||
return new MyEnumerator(myLeftIndex, myRightIndex);
|
||||
}
|
||||
|
||||
private class MyEnumerator : IEnumerator
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
using System;
|
||||
using mbts.utils;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class AppendStreamModified : ViewStream
|
||||
{
|
||||
private AppendCollection myColl = new AppendCollection ();
|
||||
private ViewStream myLeftStream, myRightstream;
|
||||
ViewStream? myLeftStream;
|
||||
ViewStream? myRightstream;
|
||||
|
||||
public AppendStreamModified (ViewStream leftStream, ViewStream rightStream)
|
||||
{
|
||||
@@ -19,7 +20,7 @@ namespace BCG
|
||||
mySetRealSource (vss[0]);
|
||||
else
|
||||
{
|
||||
AppendStreamModified rv = null;
|
||||
AppendStreamModified? rv = null;
|
||||
foreach (ViewStream s in vss)
|
||||
rv = rv == null ? new AppendStreamModified (new ViewStream[] {s}) : new AppendStreamModified (rv, s);
|
||||
mySetRealSource (rv);
|
||||
@@ -28,24 +29,26 @@ namespace BCG
|
||||
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
myLeftStream.onChange += myHandleLeftChanges;
|
||||
myRightstream.onChange += myHandleRightChanges;
|
||||
ExceptionAssert.not_null(myLeftStream).onChange += myHandleLeftChanges;
|
||||
ExceptionAssert.not_null(myRightstream).onChange += myHandleRightChanges;
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
myLeftStream.onChange -= myHandleLeftChanges;
|
||||
myRightstream.onChange -= myHandleRightChanges;
|
||||
ExceptionAssert.not_null(myLeftStream).onChange -= myHandleLeftChanges;
|
||||
ExceptionAssert.not_null(myRightstream).onChange -= myHandleRightChanges;
|
||||
}
|
||||
|
||||
private void myHandleLeftChanges (object newValue)
|
||||
{
|
||||
myFireEvent (myGetDataPacket (newValue as DataPacket, true));
|
||||
var dp = ExceptionAssert.Argument.Is<DataPacket>(newValue);
|
||||
myFireEvent(myGetDataPacket(dp, true));
|
||||
}
|
||||
|
||||
private void myHandleRightChanges (object newValue)
|
||||
{
|
||||
myFireEvent (myGetDataPacket (newValue as DataPacket, false));
|
||||
var dp = ExceptionAssert.Argument.Is<DataPacket>(newValue);
|
||||
myFireEvent(myGetDataPacket(dp, false));
|
||||
}
|
||||
|
||||
private DataPacket myGetDataPacket (DataPacket dp, bool isLeft)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
@@ -6,10 +7,10 @@ namespace BCG {
|
||||
/// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
|
||||
/// </summary>
|
||||
[Serializable]
|
||||
public class ColumnInfo : IComparable {
|
||||
public class ColumnInfo : IEquatable<ColumnInfo>, IComparable<ColumnInfo>, IComparable {
|
||||
|
||||
//the default column info does not cach values in a lookup table, therefore it is in group 0
|
||||
public ColumnInfo(string colName, int colNo, Type colType, object colValue = null) {
|
||||
public ColumnInfo(string colName, int colNo, Type colType, object? colValue = null) {
|
||||
myColName = colName;
|
||||
myColNo = colNo;
|
||||
myColType = colType;
|
||||
@@ -27,22 +28,23 @@ namespace BCG {
|
||||
type == typeof(string) ||
|
||||
type == typeof(DateTime);
|
||||
|
||||
public int CompareTo(object obj) {
|
||||
int otherColNo = ((ColumnInfo)(obj)).colNo;
|
||||
if (myColNo < otherColNo)
|
||||
return -1;
|
||||
else if (myColNo > otherColNo)
|
||||
return 1;
|
||||
return 0;
|
||||
}
|
||||
public int CompareTo(ColumnInfo? other) =>
|
||||
other is null ? 1 : colNo.CompareTo(other.colNo);
|
||||
|
||||
public override bool Equals(object obj) {
|
||||
if (obj is ColumnInfo) {
|
||||
ColumnInfo other = obj as ColumnInfo;
|
||||
return (colNo == other.colNo) && (colName == other.colName) && (colType == other.colType);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
public int CompareTo(object? obj) =>
|
||||
CompareTo(obj as ColumnInfo);
|
||||
|
||||
public bool Equals(ColumnInfo? other) =>
|
||||
other is not null
|
||||
&& colNo == other.colNo
|
||||
&& colType == other.colType
|
||||
&& colName == other.colName;
|
||||
|
||||
public override bool Equals(object? obj) =>
|
||||
Equals(obj as ColumnInfo);
|
||||
|
||||
public override int GetHashCode() =>
|
||||
HashCode.Combine(colNo, colType, colName);
|
||||
|
||||
public override string ToString() {
|
||||
return colName + " " + colNo + " " + colType;
|
||||
@@ -55,7 +57,7 @@ namespace BCG {
|
||||
private int myColNo;
|
||||
private string myColName;
|
||||
private Type myColType;
|
||||
private object myColValue;
|
||||
private object? myColValue;
|
||||
|
||||
[JsonPropertyName("colNo")]
|
||||
public int colNo { get { return myColNo; } }
|
||||
@@ -67,6 +69,6 @@ namespace BCG {
|
||||
public Type colType { get { return myColType; } }
|
||||
|
||||
[JsonIgnore]
|
||||
public object colValue {get{return myColValue;}}
|
||||
public object? colValue {get{return myColValue;}}
|
||||
}
|
||||
}
|
||||
+47
-40
@@ -1,6 +1,7 @@
|
||||
using System;
|
||||
#nullable enable
|
||||
using BCG;
|
||||
using System.Collections;
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
// common place for all serialization interfaces
|
||||
@@ -14,7 +15,7 @@ namespace bts {
|
||||
// other operations can be expressed with this
|
||||
MULTIPLE_REMOVE_ADD = 1,
|
||||
}
|
||||
public class CreateMessage {
|
||||
public class CreateMessage : IEquatable<CreateMessage> {
|
||||
public IBlueRowCollection rows;
|
||||
public CreateMessage(IBlueRowCollection rows_) {
|
||||
rows = rows_;
|
||||
@@ -30,15 +31,19 @@ namespace bts {
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
public override bool Equals(object obj) {
|
||||
if (obj != null && obj is CreateMessage) {
|
||||
var other = obj as CreateMessage;
|
||||
return BlueDataUtility.compute_equality(rows, other.rows);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public bool Equals(CreateMessage? other) =>
|
||||
other is not null && BlueDataUtility.compute_equality(rows, other.rows);
|
||||
|
||||
public override bool Equals(object? obj) =>
|
||||
Equals(obj as CreateMessage);
|
||||
|
||||
public override int GetHashCode() =>
|
||||
// Not the most elegant hash, but it works with the Equals fn. CreateMessages that are
|
||||
// 'Equal' are guaranteed to have the same hash.
|
||||
HashCode.Combine(rows.Count, rows.schema.columnInfo.Length);
|
||||
}
|
||||
public class MultipleRemoveAddMessage {
|
||||
public class MultipleRemoveAddMessage : IEquatable<MultipleRemoveAddMessage> {
|
||||
public IBlueRowCollection removed_rows;
|
||||
public IBlueRowCollection added_rows;
|
||||
public MultipleRemoveAddMessage(IBlueRowCollection removed_rows_, IBlueRowCollection added_rows_) {
|
||||
@@ -56,18 +61,23 @@ namespace bts {
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
public override bool Equals(object obj) {
|
||||
if (obj != null && obj is MultipleRemoveAddMessage) {
|
||||
MultipleRemoveAddMessage other = obj as MultipleRemoveAddMessage;
|
||||
return BlueDataUtility.compute_equality(removed_rows, other.removed_rows) &&
|
||||
BlueDataUtility.compute_equality(added_rows, other.added_rows);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
public bool Equals(MultipleRemoveAddMessage? other) =>
|
||||
other is not null
|
||||
&& BlueDataUtility.compute_equality(removed_rows, other.removed_rows)
|
||||
&& BlueDataUtility.compute_equality(added_rows, other.added_rows);
|
||||
|
||||
public override bool Equals(object? obj) =>
|
||||
Equals(obj as MultipleRemoveAddMessage);
|
||||
|
||||
public override int GetHashCode() =>
|
||||
// Not the most elegant hash, but it works with the Equals fn. CreateMessages that are
|
||||
// 'Equal' are guaranteed to have the same hash.
|
||||
HashCode.Combine(
|
||||
removed_rows.Count, removed_rows.schema.columnNames.Length, added_rows.Count, added_rows.schema.columnInfo.Length);
|
||||
}
|
||||
public StreamOp op;
|
||||
public readonly CreateMessage create_message = null;
|
||||
public readonly MultipleRemoveAddMessage multiple_remove_add_message = null;
|
||||
public readonly CreateMessage? create_message = null;
|
||||
public readonly MultipleRemoveAddMessage? multiple_remove_add_message = null;
|
||||
|
||||
public StreamIOMessage(CreateMessage create_message_) {
|
||||
op = StreamOp.CREATE;
|
||||
@@ -83,29 +93,26 @@ namespace bts {
|
||||
|
||||
public override string ToString() {
|
||||
switch (op) {
|
||||
case StreamIOMessage.StreamOp.CREATE:
|
||||
return create_message.ToString();
|
||||
case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD:
|
||||
return multiple_remove_add_message.ToString();
|
||||
case StreamOp.CREATE:
|
||||
return create_message!.ToString();
|
||||
case StreamOp.MULTIPLE_REMOVE_ADD:
|
||||
return multiple_remove_add_message!.ToString();
|
||||
}
|
||||
bts.Log.assert(false);
|
||||
return String.Empty;
|
||||
}
|
||||
public override bool Equals(object obj) {
|
||||
if (obj != null && obj is StreamIOMessage) {
|
||||
StreamIOMessage other = obj as StreamIOMessage;
|
||||
if (op == other.op) {
|
||||
switch (op) {
|
||||
case StreamOp.CREATE:
|
||||
return create_message.Equals(other.create_message);
|
||||
case StreamOp.MULTIPLE_REMOVE_ADD:
|
||||
return multiple_remove_add_message.Equals(other.multiple_remove_add_message);
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
public bool Equals(StreamIOMessage? other) =>
|
||||
op == other?.op && op switch {
|
||||
StreamOp.CREATE => create_message!.Equals(other.create_message),
|
||||
StreamOp.MULTIPLE_REMOVE_ADD => multiple_remove_add_message!.Equals(other.multiple_remove_add_message),
|
||||
_ => false,
|
||||
};
|
||||
|
||||
public override bool Equals(object? obj) =>
|
||||
Equals(obj as StreamIOMessage);
|
||||
|
||||
public override int GetHashCode() =>
|
||||
HashCode.Combine((int)op, create_message?.GetHashCode() ?? multiple_remove_add_message?.GetHashCode() ?? 0);
|
||||
}
|
||||
#region serialization
|
||||
// This helps putting stream callbacks (op + data) on the wire, to send it to the pubsubserver.
|
||||
@@ -134,4 +141,4 @@ namespace bts {
|
||||
}
|
||||
#endregion
|
||||
|
||||
}
|
||||
}
|
||||
@@ -11,45 +11,45 @@ namespace bts.BlueOrb {
|
||||
|
||||
public static class ContractUtils {
|
||||
public class FutureTradableContract {
|
||||
public string symbol;
|
||||
public string? symbol;
|
||||
public DateTime expiration;
|
||||
public Exchange exchange;
|
||||
public string contract_text;
|
||||
public string? contract_text;
|
||||
}
|
||||
public class FutureSpreadContract {
|
||||
public List<Tuple<double, FutureTradableContract>> legs;
|
||||
public string symbol {
|
||||
public List<Tuple<double, FutureTradableContract>>? legs;
|
||||
public string? symbol {
|
||||
get {
|
||||
return legs == null || legs.Count == 0 ? string.Empty : legs.ElementAt(0).Item2.symbol;
|
||||
}
|
||||
}
|
||||
public string contract_text;
|
||||
public string? contract_text;
|
||||
}
|
||||
public class FutureContinousContract {
|
||||
public string symbol;
|
||||
public string? symbol;
|
||||
public Exchange exchange;
|
||||
}
|
||||
// this is for all future conytracts belonging to a symbol
|
||||
public class FutureWildcardContract {
|
||||
public string symbol;
|
||||
public string? symbol;
|
||||
public Exchange exchange;
|
||||
}
|
||||
public class FutureContract {
|
||||
public FutureWildcardContract wildcard_contract = null;
|
||||
public FutureTradableContract future_tradable_contract = null;
|
||||
public FutureSpreadContract spread_tradable_contract = null;
|
||||
public FutureContinousContract continous_constract = null;
|
||||
public FutureWildcardContract? wildcard_contract = null;
|
||||
public FutureTradableContract? future_tradable_contract = null;
|
||||
public FutureSpreadContract? spread_tradable_contract = null;
|
||||
public FutureContinousContract? continous_constract = null;
|
||||
|
||||
public FutureContract(string txt) {
|
||||
wildcard_contract = parse_wildcard_contract(txt);
|
||||
if (wildcard_contract != null)
|
||||
return;
|
||||
FutureTradableContract fc = parse_future_contract(txt);
|
||||
var fc = parse_future_contract(txt);
|
||||
if (fc != null) {
|
||||
future_tradable_contract = fc;
|
||||
return;
|
||||
}
|
||||
FutureSpreadContract sc = parse_future_spread_contract(txt);
|
||||
var sc = parse_future_spread_contract(txt);
|
||||
if (sc != null) {
|
||||
spread_tradable_contract = sc;
|
||||
return;
|
||||
@@ -63,7 +63,7 @@ namespace bts.BlueOrb {
|
||||
public bool is_future_tradable_contract => future_tradable_contract != null;
|
||||
public bool is_spread_tradable_contract => spread_tradable_contract != null;
|
||||
|
||||
public string symbol() {
|
||||
public string? symbol() {
|
||||
if (wildcard_contract != null)
|
||||
return wildcard_contract.symbol;
|
||||
if (continous_constract != null) {
|
||||
@@ -76,7 +76,7 @@ namespace bts.BlueOrb {
|
||||
Log.assert(false);
|
||||
return string.Empty;
|
||||
}
|
||||
public string contract_text {
|
||||
public string? contract_text {
|
||||
get {
|
||||
if (future_tradable_contract != null)
|
||||
return future_tradable_contract.contract_text;
|
||||
@@ -87,7 +87,7 @@ namespace bts.BlueOrb {
|
||||
}
|
||||
}
|
||||
#region Parsing
|
||||
public static FutureWildcardContract parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) {
|
||||
public static FutureWildcardContract? parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) {
|
||||
if (contract_txt.EndsWith("*")) {
|
||||
contract_txt = contract_txt.Replace("*", string.Empty);
|
||||
return new FutureWildcardContract() {
|
||||
@@ -97,7 +97,7 @@ namespace bts.BlueOrb {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
public static FutureTradableContract parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) {
|
||||
public static FutureTradableContract? parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) {
|
||||
// Validate the future string length
|
||||
if (!contract_txt.All(c => char.IsLetterOrDigit(c)))
|
||||
return null;
|
||||
@@ -153,7 +153,7 @@ namespace bts.BlueOrb {
|
||||
return futureContract;
|
||||
}
|
||||
// Method to parse a futures spread, e.g. "CLX4-CLZ4"
|
||||
public static FutureSpreadContract parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) {
|
||||
public static FutureSpreadContract? parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) {
|
||||
// Split the string by '-' to handle each leg
|
||||
var spreadParts = contract_txt.Split('-');
|
||||
if (spreadParts.Length < 2) {
|
||||
@@ -201,11 +201,11 @@ namespace bts.BlueOrb {
|
||||
#endregion
|
||||
|
||||
public static ContractType get_type(string contract_txt) {
|
||||
FutureTradableContract fc = parse_future_contract(contract_txt);
|
||||
var fc = parse_future_contract(contract_txt);
|
||||
if (fc != null) {
|
||||
return ContractType.Future;
|
||||
}
|
||||
FutureSpreadContract sc = parse_future_spread_contract(contract_txt);
|
||||
var sc = parse_future_spread_contract(contract_txt);
|
||||
if (sc != null) {
|
||||
return ContractType.FutureSpread;
|
||||
}
|
||||
@@ -220,13 +220,13 @@ namespace bts.BlueOrb {
|
||||
}
|
||||
|
||||
public static Tuple<ContractCategory, string> get_symbol_category(string symbol) {
|
||||
FutureTradableContract fc = parse_future_contract(symbol);
|
||||
var fc = parse_future_contract(symbol);
|
||||
if (fc != null) {
|
||||
return Tuple.Create(ContractCategory.FutureString, fc.symbol);
|
||||
return Tuple.Create(ContractCategory.FutureString, fc.symbol ?? string.Empty);
|
||||
}
|
||||
var sc = parse_future_spread_contract(symbol);
|
||||
if (sc != null) {
|
||||
return Tuple.Create(ContractCategory.FutureString, sc.symbol);
|
||||
return Tuple.Create(ContractCategory.FutureString, sc.symbol ?? string.Empty);
|
||||
}
|
||||
return Tuple.Create(ContractCategory.Symbol, symbol);
|
||||
}
|
||||
|
||||
+9
-11
@@ -1,8 +1,6 @@
|
||||
namespace bts.BlueOrb {
|
||||
using bts.SilverPine;
|
||||
using BCG;
|
||||
|
||||
|
||||
public class DateTimeRange {
|
||||
public readonly DateTime range_beg;
|
||||
public readonly DateTime range_end;
|
||||
@@ -18,7 +16,7 @@ namespace bts.BlueOrb {
|
||||
}
|
||||
|
||||
public class FutureBarsLoader {
|
||||
public static string get_db_table_name(BarSize bar_size, string symbol, bool is_continous) {
|
||||
public static string? get_db_table_name(BarSize bar_size, string symbol, bool is_continous) {
|
||||
switch (bar_size) {
|
||||
case BarSize.ONE_DAY: {
|
||||
if (is_continous) {
|
||||
@@ -68,22 +66,22 @@ namespace bts.BlueOrb {
|
||||
string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
|
||||
return new SQLiteStream(SQLiteHelper.db_name, sql_str);
|
||||
};
|
||||
string symbol = future_contract.symbol();
|
||||
var symbol = future_contract.symbol() ?? string.Empty;
|
||||
if (future_contract.wildcard_contract != null) {
|
||||
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
|
||||
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, false) ?? string.Empty;
|
||||
ViewStream rs = get_sqlite_str(table_name);
|
||||
//rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
|
||||
mySetRealSource(rs);
|
||||
return;
|
||||
}
|
||||
if (future_contract.is_continous_contract) {
|
||||
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true);
|
||||
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true) ?? string.Empty;
|
||||
ViewStream rs = get_sqlite_str(table_name);
|
||||
//rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
|
||||
mySetRealSource(rs);
|
||||
return;
|
||||
} else {
|
||||
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
|
||||
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, false) ?? string.Empty;
|
||||
ViewStream rs = get_sqlite_str(table_name);
|
||||
//rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
|
||||
//TODO: filter it
|
||||
@@ -115,20 +113,20 @@ namespace bts.BlueOrb {
|
||||
string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
|
||||
return new SQLiteStream(SQLiteHelper.db_name, sql_str);
|
||||
};
|
||||
string symbol = future_contract.symbol();
|
||||
var symbol = future_contract.symbol() ?? string.Empty;
|
||||
if (future_contract.wildcard_contract != null) {
|
||||
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
|
||||
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, false) ?? string.Empty;
|
||||
ViewStream rs = get_sqlite_str(table_name);
|
||||
mySetRealSource(rs);
|
||||
return;
|
||||
}
|
||||
if (future_contract.is_continous_contract) {
|
||||
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true);
|
||||
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true) ?? string.Empty;
|
||||
ViewStream rs = get_sqlite_str(table_name);
|
||||
mySetRealSource(rs);
|
||||
return;
|
||||
} else {
|
||||
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
|
||||
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, false) ?? string.Empty;
|
||||
ViewStream rs = get_sqlite_str(table_name);
|
||||
// TODO: filter
|
||||
rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'");
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
namespace bts {
|
||||
using BCG;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
@@ -8,7 +10,7 @@ namespace bts {
|
||||
using System.Text.Json.Nodes;
|
||||
[DataStreamUserRespresentation("benzingaNews")]
|
||||
public class BenzingaNewsStream : ViewStream {
|
||||
private IBlueTableSchema _schema = null;
|
||||
private IBlueTableSchema? _schema;
|
||||
private int _id = 0;
|
||||
|
||||
|
||||
@@ -63,14 +65,14 @@ namespace bts {
|
||||
}
|
||||
foreach (JsonNode jnode in json_objs) {
|
||||
if (jnode is JsonObject json_object) {
|
||||
string symbol = jnode["symbol"] == null ? String.Empty : jnode["symbol"].ToString();
|
||||
string exchange = jnode["exchange"] == null ? String.Empty : jnode["exchange"].ToString();
|
||||
string symbol = jnode["symbol"]?.ToString() ?? string.Empty;
|
||||
string exchange = jnode["exchange"]?.ToString() ?? string.Empty;
|
||||
MutableBlueRow nr = new MutableBlueRow(_schema, r);
|
||||
nr.setValue("id", _id++);
|
||||
nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
|
||||
nr.setValue("symbol", symbol);
|
||||
nr.setValue("exchange", exchange);
|
||||
nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
|
||||
nr.setValue("is_primary_exchange", bool.Parse(ExceptionAssert.not_null(jnode["primary"]).ToString()));
|
||||
expanded_rows.Add(nr);
|
||||
}
|
||||
}
|
||||
@@ -84,4 +86,4 @@ namespace bts {
|
||||
mySetRealSource(output_stream);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+7
-9
@@ -2,8 +2,6 @@ namespace bts {
|
||||
using BCG;
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
|
||||
//[DataStreamUserRespresentation("benzingaNewsSymbols")]
|
||||
public class BenzingaNewsSymbolStateStream : ViewStream, INonIncGroupByEvaluator {
|
||||
@@ -11,23 +9,23 @@ namespace bts {
|
||||
ViewStream r = new NonIncGroupByStream(news_str, this);
|
||||
mySetRealSource(r);
|
||||
}
|
||||
public BenzingaNewsSymbolStateStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl) :
|
||||
this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) {
|
||||
}
|
||||
|
||||
public BenzingaNewsSymbolStateStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl)
|
||||
: this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) { }
|
||||
|
||||
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
|
||||
BlueTable rv = null;
|
||||
BlueTable? rv = null;
|
||||
foreach (var kv in grouped_rows) {
|
||||
var symbol_rows = kv.Value;
|
||||
IBlueRow state_row = BlueDataUtility.find_max(symbol_rows, "created_at");
|
||||
rv = rv == null ? new BlueTable(state_row.schema) : rv;
|
||||
rv ??= new(state_row.schema);
|
||||
rv.addRow(state_row);
|
||||
}
|
||||
return rv;
|
||||
return rv ?? BlueTable.Empty("BenzingaNewsSymbolState");
|
||||
}
|
||||
|
||||
public string[] get_group_by_cols() {
|
||||
return new string[] { "symbol" };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ using bts.config;
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace BCG {
|
||||
|
||||
@@ -21,7 +22,9 @@ namespace BCG {
|
||||
}
|
||||
public class RenkoCalculator {
|
||||
public readonly double brick_size;
|
||||
public RenkoBrick current_brick = null;
|
||||
public RenkoBrick? current_brick = null;
|
||||
|
||||
[MemberNotNull(nameof(current_brick))]
|
||||
private bool _add_price(double price) {
|
||||
if (current_brick == null) {
|
||||
current_brick = new RenkoBrick {
|
||||
@@ -309,4 +312,4 @@ namespace BCG {
|
||||
return wma_values;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
-2
@@ -3,6 +3,8 @@
|
||||
|
||||
using bts.md;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
@@ -13,7 +15,7 @@
|
||||
public class SpatialArbStatsStream : ViewStream{
|
||||
private ViewStream _spatial_pairs_stream;
|
||||
private ViewStreamDatabase _vsd;
|
||||
private ViewStream _tot_str = null;
|
||||
private ViewStream? _tot_str = null;
|
||||
private HashSet<string> _key_map = new HashSet<string>();
|
||||
private ViewStream _bbo_stream;
|
||||
private readonly TimeSpan _window_sz, _sampling_period_sz;
|
||||
@@ -89,7 +91,7 @@
|
||||
//_tot_str = new SortStream(_tot_str, "-n_trade_ticks");
|
||||
_tot_str.onChange += _handle_tot_stream;
|
||||
|
||||
DataPacket dp = dp_obj as DataPacket;
|
||||
var dp = ExceptionAssert.Argument.Is<DataPacket>(dp_obj);
|
||||
foreach (IBlueRow r in dp.newRows) {
|
||||
string s1 = r.get<string>("s1");
|
||||
string s2 = r.get<string>("s2");
|
||||
|
||||
+3
-1
@@ -4,6 +4,8 @@
|
||||
using bts.api;
|
||||
using bts.md;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
@@ -67,7 +69,7 @@
|
||||
|
||||
private void _main_stream_onchange(object dp_obj) {
|
||||
_key_map.Clear();
|
||||
DataPacket dp = dp_obj as DataPacket;
|
||||
var dp = ExceptionAssert.Argument.Is<DataPacket>(dp_obj);
|
||||
BlueTable table = new BlueTable(_schema);
|
||||
var rows = dp.newRows.to_list();
|
||||
foreach (var r1 in rows) {
|
||||
|
||||
+6
-29
@@ -1,30 +1,7 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MathNet.Numerics.Statistics;
|
||||
using MathNet.Numerics.LinearRegression;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MathNet.Numerics.Statistics;
|
||||
using MathNet.Numerics.LinearAlgebra;
|
||||
using Amazon.Runtime;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MathNet.Numerics.Statistics;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MathNet.Numerics.Statistics;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MathNet.Numerics.Statistics;
|
||||
// stats calculator
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MathNet.Numerics.Statistics;
|
||||
|
||||
/// <summary>
|
||||
/// Provides evaluation metrics to assess the quality of a synthetic spread for mean-reversion trading.
|
||||
@@ -35,13 +12,13 @@ public static class SpreadPerformanceEvaluator
|
||||
|
||||
public static class SpreadStatistics {
|
||||
public class SpreadStatsResult {
|
||||
public Dictionary<string, double> bid_zscore_distribution { get; set; }
|
||||
public Dictionary<string, double> ask_zscore_distribution { get; set; }
|
||||
public Dictionary<string, double> mid_zscore_distribution { get; set; }
|
||||
public Dictionary<string, double> bid_zscore_distribution { get; set; } = new(0);
|
||||
public Dictionary<string, double> ask_zscore_distribution { get; set; } = new(0);
|
||||
public Dictionary<string, double> mid_zscore_distribution { get; set; } = new(0);
|
||||
|
||||
public DescriptiveStatistics bid_stats { get; set; }
|
||||
public DescriptiveStatistics ask_stats { get; set; }
|
||||
public DescriptiveStatistics mid_stats { get; set; }
|
||||
public DescriptiveStatistics bid_stats { get; set; } = new();
|
||||
public DescriptiveStatistics ask_stats { get; set; } = new();
|
||||
public DescriptiveStatistics mid_stats { get; set; } = new();
|
||||
}
|
||||
|
||||
private static readonly string[] _zscore_labels = new[]
|
||||
|
||||
+4
-2
@@ -1,10 +1,12 @@
|
||||
namespace bts.BlueOrb {
|
||||
using BCG;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using System.Diagnostics;
|
||||
|
||||
public class SpreadPerformanceStream : ViewStream {
|
||||
ViewStream _spread_price_stream = null;
|
||||
ViewStream _spread_price_stream;
|
||||
private readonly IBlueTableSchema _schema;
|
||||
private readonly int _window_secs;
|
||||
private readonly double _buy_threshold, _sell_threshold, _flat_threshold;
|
||||
@@ -70,7 +72,7 @@ namespace bts.BlueOrb {
|
||||
}
|
||||
|
||||
private void _handle_spread_prices(object dp_obj) {
|
||||
DataPacket dp = dp_obj as DataPacket;
|
||||
var dp = ExceptionAssert.Argument.Is<DataPacket>(dp_obj);
|
||||
List<Tuple<DateTime, double, double, double, double>> spread_ts_prices = new List<Tuple<DateTime, double, double, double, double>>();
|
||||
DateTime min_time = DateTime.MaxValue; DateTime max_time = DateTime.MinValue;
|
||||
var sw = Stopwatch.StartNew();
|
||||
|
||||
@@ -5,6 +5,8 @@ namespace bts.BlueOrb{
|
||||
using bts;
|
||||
using bts.md;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using Spark.Core.Disposables;
|
||||
using Spark.Core.Reactive;
|
||||
|
||||
@@ -103,11 +105,11 @@ namespace bts.BlueOrb{
|
||||
new_state.addRow(r);
|
||||
}
|
||||
}
|
||||
DataPacket dp = null;
|
||||
DataPacket? dp;
|
||||
if (myLastValue == null) {
|
||||
dp = new DataPacket(new_state, new_state, new BlueTable(_bbo_schema));
|
||||
} else {
|
||||
DataPacket old_dp = myLastValue as DataPacket;
|
||||
var old_dp = ExceptionAssert.of_type<DataPacket>(myLastValue);
|
||||
IBlueRowCollection added_rows, del_rows;
|
||||
BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
|
||||
dp = new DataPacket(new_state, added_rows, del_rows);
|
||||
|
||||
+1
-1
@@ -26,7 +26,7 @@ namespace bts.BlueOrb{
|
||||
|
||||
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
|
||||
Func<string, bool> is_future = (c) => ContractUtils.get_category(c) == ContractUtils.ContractCategory.FutureString;
|
||||
BlueTable rv = null;
|
||||
BlueTable? rv = null;
|
||||
Func<List<IBlueRow>, IBlueRow> reduce = (rows) => {
|
||||
var future_rows = rows.Where(r => is_future(r.get<string>("contract")));
|
||||
IBlueRow rv = future_rows.ElementAt(0);
|
||||
|
||||
@@ -4,8 +4,12 @@ namespace bts.BlueOrb {
|
||||
using bts;
|
||||
using bts.api;
|
||||
using bts.md;
|
||||
|
||||
using BCG;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using Spark.Core;
|
||||
using Spark.Core.Disposables;
|
||||
using Spark.Core.Reactive;
|
||||
|
||||
@@ -13,18 +17,14 @@ namespace bts.BlueOrb {
|
||||
|
||||
using System.Collections.Generic;
|
||||
|
||||
using System.Linq;
|
||||
using System.Reactive.Linq;
|
||||
|
||||
using SecurityKey = System.UInt32;
|
||||
|
||||
[DataStreamUserRespresentation("InstrumentsStream")]
|
||||
public class InstrumentsStream : ViewStream, IWorker {
|
||||
private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
|
||||
private DateTime _last_poll_time = DateTime.MinValue;
|
||||
private ViewStreamDatabase _vsd;
|
||||
private int _id = 0;
|
||||
IReadOnlyDictionary<uint, SecurityImmutable> _cached_state = null;
|
||||
IReadOnlyDictionary<uint, SecurityImmutable>? _cached_state;
|
||||
MarketDataClient _md_client;
|
||||
private bool _dirty = false;
|
||||
private readonly IBlueTableSchema _instrument_schema = BlueDataUtility.form_schema("instruments",
|
||||
@@ -68,7 +68,7 @@ namespace bts.BlueOrb {
|
||||
_dirty = true;
|
||||
//_log_updates(state);
|
||||
// just cacching in here, will populate the rows in _poll_scan and fire the datapacket
|
||||
bts.Log.Gen.info($"InstrumentsStream._on_instruments state: {state.Count()}");
|
||||
bts.Log.Gen.info($"InstrumentsStream._on_instruments state: {state.Count}");
|
||||
_cached_state = state;
|
||||
}
|
||||
|
||||
@@ -85,13 +85,13 @@ namespace bts.BlueOrb {
|
||||
return;
|
||||
BlueTable new_state = new BlueTable(_instrument_schema);
|
||||
foreach (var kv in _cached_state) {
|
||||
SecurityKey skey = kv.Key;
|
||||
var skey = kv.Key;
|
||||
int id = Convert.ToInt32(skey);
|
||||
SecurityImmutable sec = kv.Value;
|
||||
bts.api.SecurityType sec_type = sec.security_type();
|
||||
object exp = DBNull.Value;
|
||||
if (sec_type == bts.api.SecurityType.Future || sec_type == bts.api.SecurityType.Option)
|
||||
exp = sec.security_expiration();
|
||||
exp = sec.security_expiration().assert_value();
|
||||
object strike = DBNull.Value;
|
||||
if (sec_type == bts.api.SecurityType.Option)
|
||||
strike = sec.strike();
|
||||
@@ -101,11 +101,11 @@ namespace bts.BlueOrb {
|
||||
IBlueRow r = new BlueRow(_instrument_schema, id, skey, sec.security_desc(), sec.security_type().ToString(), sec.display_exchange().ToString(), sec.Symbol, sec.Currency, sec.ContractSpec.CashPerPoint, exp, strike, cp);
|
||||
new_state.addRow(r);
|
||||
}
|
||||
DataPacket dp = null;
|
||||
DataPacket? dp;
|
||||
if (myLastValue == null) {
|
||||
dp = new DataPacket(new_state, new_state, new BlueTable(_instrument_schema));
|
||||
} else {
|
||||
DataPacket old_dp = myLastValue as DataPacket;
|
||||
var old_dp = ExceptionAssert.of_type<DataPacket>(myLastValue);
|
||||
IBlueRowCollection added_rows, del_rows;
|
||||
BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
|
||||
dp = new DataPacket(new_state, added_rows, del_rows);
|
||||
|
||||
@@ -167,8 +167,8 @@ namespace bts.BlueOrb {
|
||||
return String.Empty;
|
||||
}
|
||||
|
||||
private Source _sourse = Source.SQL; // get from config
|
||||
private readonly string _location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:\\BlueOrb_Data\\":Path.Combine(Environment.GetEnvironmentVariable("HOME") , "BlueOrb_Data");
|
||||
// private Source _sourse = Source.SQL; // get from config
|
||||
// private readonly string _location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:\\BlueOrb_Data\\":Path.Combine(Environment.GetEnvironmentVariable("HOME") , "BlueOrb_Data");
|
||||
private readonly string _db_name = "stream_data.db";
|
||||
// the symbol is used ot retreive the correct table, the data flows as-is from the table
|
||||
// with some name / type changes
|
||||
|
||||
@@ -1,44 +1,28 @@
|
||||
namespace BCG {
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using bts.api;
|
||||
using bts.api.domain;
|
||||
using bts.api.synthetic;
|
||||
using bts;
|
||||
using bts.api.core;
|
||||
using bts.md;
|
||||
using bts.spark;
|
||||
using bts.utils;
|
||||
|
||||
using mbts;
|
||||
using mbts.co2;
|
||||
using mbts.config;
|
||||
using mbts.spark;
|
||||
using mbts.ui;
|
||||
using mbts.utils;
|
||||
|
||||
using Spark.Core.Data;
|
||||
using Spark.Core.Disposables;
|
||||
using Spark.Core.Reactive;
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Linq;
|
||||
using System.Reactive.Linq;
|
||||
using bts.api.core;
|
||||
using Spark.Api.MarketData;
|
||||
|
||||
using SecurityKey = System.UInt32;
|
||||
using System.Data;
|
||||
using System.Reflection.Metadata;
|
||||
using bts;
|
||||
|
||||
[DataStreamUserRespresentation("price_stats")]
|
||||
public class PriceStatsStream : ViewStream, IWorker {
|
||||
private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
|
||||
private DateTime _last_poll_time = DateTime.MinValue;
|
||||
private ViewStreamDatabase _vsd;
|
||||
private int _id = 0;
|
||||
IReadOnlyDictionary<SecurityKey, PriceStats> _cached_state = null;
|
||||
IReadOnlyDictionary<uint, PriceStats>? _cached_state = null;
|
||||
MarketDataClient _md_client;
|
||||
private bool _dirty = false;
|
||||
private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo",
|
||||
@@ -52,14 +36,13 @@ namespace BCG {
|
||||
IDisposable? subscription;
|
||||
|
||||
// TODO: add cons for symbol, exchange etc
|
||||
public PriceStatsStream()
|
||||
: this(ViewStreamDatabase.instance) {
|
||||
}
|
||||
public PriceStatsStream(MarketDataClient mdClient)
|
||||
: this(mdClient, ViewStreamDatabase.instance) { }
|
||||
|
||||
public PriceStatsStream(ViewStreamDatabase vsd) {
|
||||
public PriceStatsStream(MarketDataClient mdClient, ViewStreamDatabase vsd) {
|
||||
bts.Log.Gen.info("PriceStatsStream.cons");
|
||||
_md_client = mdClient;
|
||||
_vsd = vsd;
|
||||
_md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
@@ -69,7 +52,7 @@ namespace BCG {
|
||||
}
|
||||
|
||||
private void _log_updates(TableUpdate<PriceStats, uint> update) {
|
||||
int i = 0;
|
||||
// int i = 0;
|
||||
// foreach (BookTuple bt in update.added) {
|
||||
// bts.Log.Gen.info($"PriceStatsStream added: idx: {i++} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
|
||||
// }
|
||||
@@ -99,18 +82,18 @@ namespace BCG {
|
||||
return;
|
||||
BlueTable new_state = new BlueTable(_pstats_scheam);
|
||||
foreach (var kv in _cached_state) {
|
||||
SecurityKey skey = kv.Key;
|
||||
var skey = kv.Key;
|
||||
int id = Convert.ToInt32(skey);
|
||||
PriceStats pstats = kv.Value;
|
||||
var today = DateTime.Now.Date;
|
||||
IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), today, pstats.open, pstats.close, pstats.settlement);
|
||||
new_state.addRow(r);
|
||||
}
|
||||
DataPacket dp = null;
|
||||
DataPacket? dp;
|
||||
if (myLastValue == null) {
|
||||
dp = new DataPacket(new_state, new_state, new BlueTable(_pstats_scheam));
|
||||
} else {
|
||||
DataPacket old_dp = myLastValue as DataPacket;
|
||||
var old_dp = ExceptionAssert.of_type<DataPacket>(myLastValue);
|
||||
IBlueRowCollection added_rows, del_rows;
|
||||
BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
|
||||
dp = new DataPacket(new_state, added_rows, del_rows);
|
||||
|
||||
@@ -85,7 +85,8 @@ namespace bts.BlueOrb {
|
||||
.GetProperty("choices")[0]
|
||||
.GetProperty("message")
|
||||
.GetProperty("content")
|
||||
.GetString();
|
||||
.GetString()
|
||||
?? string.Empty;
|
||||
|
||||
//return gpt_response;
|
||||
DateTime t1 = DateTime.Now;
|
||||
|
||||
+3
-3
@@ -1,5 +1,6 @@
|
||||
using BCG.Utilities.General;
|
||||
using BCG;
|
||||
using mbts.utils;
|
||||
|
||||
namespace bts {
|
||||
/// <summary>
|
||||
@@ -10,7 +11,7 @@ namespace bts {
|
||||
public class PubSubServerStateStream : ViewStream, IWorker {
|
||||
private IDictionary<string, TagPubSubInfo> _tag_map = new Dictionary<string, TagPubSubInfo>();
|
||||
private readonly string _tag = "System_AllSubscriptions";
|
||||
private IBlueTableSchema _schema = null;
|
||||
private IBlueTableSchema? _schema;
|
||||
|
||||
private SubscriberCore _sub_core;
|
||||
private ViewStreamDatabase _vsd;
|
||||
@@ -47,12 +48,11 @@ namespace bts {
|
||||
var updates = _sub_core.get_updates(TimeSpan.Zero);
|
||||
//this is where the pubsub updates are converted to rows
|
||||
if (updates.Count() > 0) {
|
||||
int id = 0;
|
||||
foreach (var u in updates) {
|
||||
if (u.tag != _tag) {
|
||||
continue;
|
||||
}
|
||||
var tpsi_list = TagPubSubInfo.deserialize(u.data);
|
||||
var tpsi_list = TagPubSubInfo.deserialize(Log.notnull(u.data));
|
||||
foreach (var tsi in tpsi_list) {
|
||||
_tag_map[tsi.tag] = tsi;
|
||||
}
|
||||
|
||||
@@ -7,8 +7,8 @@ using BCG.Utilities.General;
|
||||
public class PubsubHeartbeatStream : ViewStream, IWorker {
|
||||
private ViewStreamDatabase _vsd;
|
||||
private SubscriberCore _subcore = new SubscriberCore();
|
||||
private MutableBlueRow _row = null;
|
||||
private readonly IBlueTableSchema _schema = null;
|
||||
private MutableBlueRow? _row;
|
||||
private readonly IBlueTableSchema? _schema;
|
||||
public PubsubHeartbeatStream(ViewStreamDatabase vsd) {
|
||||
_vsd = vsd;
|
||||
_schema = BlueDataUtility.form_schema("PS_HBB",
|
||||
|
||||
@@ -5,12 +5,14 @@ namespace bts
|
||||
|
||||
using Bts.DataStreamCore.StreamFactories;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
public class TableMetaDataStream : ViewStream, IWorker
|
||||
{
|
||||
private class TableState
|
||||
{
|
||||
public ViewStream table_stream;
|
||||
public IBlueTableSchema schema = null;
|
||||
public IBlueTableSchema? schema;
|
||||
public bool processed = false;
|
||||
public TableState(ViewStream table_stream_)
|
||||
{
|
||||
@@ -19,8 +21,7 @@ namespace bts
|
||||
}
|
||||
private void _handle_table_update(object dp_obj)
|
||||
{
|
||||
DataPacket dp = dp_obj as DataPacket;
|
||||
schema = dp.schema;
|
||||
schema = ExceptionAssert.Argument.Is<DataPacket>(dp_obj).schema;
|
||||
}
|
||||
public void unsub()
|
||||
{
|
||||
@@ -93,7 +94,7 @@ namespace bts
|
||||
|
||||
private void _handle_base_stream(object dp_obj)
|
||||
{
|
||||
DataPacket dp = dp_obj as DataPacket;
|
||||
var dp = ExceptionAssert.Argument.Is<DataPacket>(dp_obj);
|
||||
foreach (IBlueRow r in dp.newRows)
|
||||
{
|
||||
string tag = r.get<string>("tag");
|
||||
@@ -108,12 +109,7 @@ namespace bts
|
||||
{
|
||||
Log.assert(tag_name.StartsWith("DS_"));
|
||||
string table_name = tag_name.Remove(0, 3);
|
||||
TableState ts = null;
|
||||
if (!_table_map.TryGetValue(table_name, out ts))
|
||||
{
|
||||
ts = new TableState(tableStreamFactory.Create(table_name));
|
||||
_table_map.Add(table_name, ts);
|
||||
}
|
||||
_table_map.GetOrAdd(table_name, _ => new(tableStreamFactory.Create(table_name)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user