cleanup: Fix a ton of build warnings (cs)

All these warnings have been bothering me for a while. The work isn't done, but
this commit chops them down from ~1000 to ~600.

* AppendCollection.cs
* AppendStreamModified.cs
* BenzingaNewsStream.cs
* BenzingaNewsSymbolStateStream.cs
* BookStream.cs
* CSVSaverStream.cs
* ChartsUtils.cs
* ColumnInfo.cs
* ContractUtils.cs
* ContractUtils.cs
* DiffStream.cs
* FuturesActiveContractStream.cs
* FuzzyJoinOverTimeStream.cs
* HistoricalBarsStream.cs
* HistoricalBarsStream.cs
* InstrumentsStream.cs
* JoinOperandRowIndex.cs
* JoinRow.cs
* JoinStream.cs
* JoinTable.cs
* JoinTableSchema.cs
* LatencyStream.cs
* LeftInternalJoinStream.cs
* LightForeignJoinRow.cs
* LightForeignJoinRowList.cs
* LightLeftForeignJoinRow.cs
* LiteralStream.cs
* NLStream.cs
* NLToDSLStream.cs
* NLToIndexedFewShotPromptStream.cs
* NonIncGroupByStream.cs
* NonIncJoinBookkeeper.cs
* NonIncJoinSpec.cs
* OHLCVStream.cs
* OpenAIPromptStream.cs
* PriceStatsStream.cs
* PriceStatsStream.cs
* ProjectHelper.cs
* ProjectStream.cs
* PubSubServerStateStream.cs
* PubsubHeartbeatStream.cs
* SimpleNode.cs
* SortRow.cs
* SpatialArbStatsStream.cs
* SpatialFuturePairsStream.cs
* SpreadPerformanceEvaluator.cs
* SpreadPerformanceStream.cs
* StreamIOCommon.cs
* SuperJoinStream.cs
* TableMetaDataStream.cs
* TextWriterStream.cs
* TimeAverageStream.cs
This commit is contained in:
John Colvin committed 2025-11-19 13:57:47 -05:00
1 parent 7418f5b5a1
commit a4efaa0089
48 files changed
+610 -590

No files matched your search

@@ -1,5 +1,4 @@
using System;
using System.Collections;
using System.Collections.Generic;
using mbts;
@@ -12,30 +11,21 @@ namespace BCG
{
ViewStream myLStream, myRStream;
DataPacket myLastR = null, myLastL = null;
DataPacket? myLastR = null, myLastL = null;
BlueTable myLastFired = null;
BlueTable? myLastFired = null;
double myThreshold = .00005;
string[] myColumnsToMatch;
public DiffStream(ViewStream v1, ViewStream v2) : this(null, v1, v2, "id") { }
public DiffStream(ViewStream v1, ViewStream v2, params string[] ColumnsToMatch) : this(null, v1, v2, ColumnsToMatch) { }
public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) :
this(null, v1, v2, thresh, ColumnsToMatch) { }
public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
{
public DiffStream(ViewStream v1, ViewStream v2) : this(v1, v2, "id") { }
public DiffStream(ViewStream v1, ViewStream v2, params string[] columnsToMatch) : this(v1, v2, .00005, columnsToMatch) { }
public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch) {
myThreshold = thresh;
myLStream = v1;
myRStream = v2;
myColumnsToMatch = columnsToMatch;
}
public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
: this(db,v1,v2,.00005,columnsToMatch)
{
}
protected override void myOnFirstSubscriber()
{
@@ -52,8 +42,14 @@ namespace BCG
struct ListComparer : IEqualityComparer<List<object>>
{
public bool Equals(List<object> x, List<object> y)
public bool Equals(List<object>? x, List<object>? y)
{
if (x is null) {
return y is null;
}
if (y is null) {
return false;
}
if (x.Count != y.Count)
return false;
@@ -74,21 +70,20 @@ namespace BCG
void myHandleCallbacks()
{
DataPacket packet = null;
DataPacket? packet;
if (myLastL == null || myLastR == null)
{
packet = myLastL == null ? myLastR : myLastL;
if (packet != null)
if (packet != null) {
myFireEvent(packet);
myLastFired = new BlueTable(packet.newRows);
myLastFired = new BlueTable(packet.newRows);
}
return;
}
BlueTable table = new BlueTable(myLastL.newRows.schema);
Dictionary<List<object>, Pair<BlueRow, BlueRow>> rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(new ListComparer());
Dictionary<List<object>, Pair<BlueRow?, BlueRow?>> rowPairs = new(new ListComparer());
foreach(BlueRow row in myLastL.newRows)
{
@@ -104,7 +99,7 @@ namespace BCG
}
}
rowPairs[hash] = new Pair<BlueRow, BlueRow>(row, null);
rowPairs[hash] = new(row, null);
}
foreach (BlueRow row in myLastR.newRows)
@@ -119,12 +114,11 @@ namespace BCG
catch (Exception) { }
}
Pair<BlueRow, BlueRow> pair;
rowPairs.TryGetValue(hash, out pair);
rowPairs.TryGetValue(hash, out var pair);
if (pair == null)
pair = new Pair<BlueRow, BlueRow>(null, row);
pair = new(null, row);
else
pair.second = row;
@@ -132,13 +126,11 @@ namespace BCG
}
int id = 0;
foreach (KeyValuePair<List<object>, Pair<BlueRow, BlueRow>> hashpair in rowPairs)
foreach (var (hashVals, pair) in rowPairs)
{
Pair<BlueRow, BlueRow> pair = hashpair.Value;
if (pair.first == null || pair.second == null)
{
BlueRow row = pair.first == null ? pair.second : pair.first;
var row = pair.first ?? pair.second ?? throw new InvalidOperationException("Both rows in pair are null");
Dictionary<string, object> newRow = new Dictionary<string, object>();
foreach (ColumnInfo info in row.schema.columnInfo)
@@ -211,4 +203,4 @@ namespace BCG
}
}
}
@@ -1,4 +1,8 @@
using System;
using bts;
using Bts.DataStreamCore;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
@@ -40,12 +44,12 @@ namespace BCG
private IComparer<IBlueRow> myRightCompare;
protected FuzzyRowComparer myFuzzyCompare;
protected TimeSpan myTimeWindow;
protected IBlueTableSchema myLeftSchema;
protected IBlueTableSchema myRightSchema;
protected IBlueTableSchema myOutputSchema;
protected IBlueTableSchema? myLeftSchema;
protected IBlueTableSchema? myRightSchema;
protected IBlueTableSchema? myOutputSchema;
private int myID = 0;
protected TimeWindowTable myStateRows;
protected TimeWindowTable? myStateRows;
private DateTime myNextCull;
const long MinCullIntervalMS = 1000;
@@ -65,7 +69,7 @@ namespace BCG
this.myFuzzyCompare = fuzzyCompare;
}
public virtual IBlueTableSchema OutputSchema
public virtual IBlueTableSchema? OutputSchema
{
get
{
@@ -150,7 +154,7 @@ namespace BCG
int count;
for (count = 0; count < list.Count; count++)
{
if ((DateTime)list[count][timeColumn] > expiry)
if (list[count].GetValue<DateTime>(timeColumn) > expiry)
{
// Stop when we find the first row that we can keep. Since this is sorted all others are also kept.
break;
@@ -172,14 +176,17 @@ namespace BCG
this.myFitness = fitness;
}
public IBlueRow myRightRow { get; private set; }
public IBlueRow myLeftRow { get; set; }
public IBlueRow? myLeftRow { get; set; }
public double myFitness { get; private set; }
public IBlueRow myMergedRow { get; set; }
public IBlueRow? myMergedRow { get; set; }
#region IComparable<RowMatch> Members
public int CompareTo(RowMatch other)
public int CompareTo(RowMatch? other)
{
if (other is null) {
return 1;
}
return myFitness.CompareTo(other.myFitness);
}
@@ -262,6 +269,10 @@ namespace BCG
private void HandleCorrection(List<RowMatch> corrections, BlueTable adds, BlueTable dels)
{
if (corrections.Count == 0) {
return;
}
Log.notnull(myStateRows);
// We already emitted correction.MergedRow so we need to redact that and
// then emit correction.left as an add since it is no longer matched
foreach (RowMatch correction in corrections.Distinct())
@@ -285,7 +296,7 @@ namespace BCG
int r = 0;
for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; )
{
if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn])
if (leftExpired[l].GetValue<DateTime>(myLeftTimeColumn) <= rightExpired[r].GetValue<DateTime>(myRightTimeColumn))
{
// Advance Left
IEnumerable<RowMatch> rm = rightMatches[l];
@@ -332,7 +343,7 @@ namespace BCG
private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable<RowMatch> rightMatches)
{
IBlueRow merged = null;
IBlueRow? merged = null;
if (rightMatches != null)
{
foreach (RowMatch match in rightMatches)
@@ -366,9 +377,9 @@ namespace BCG
/// Override this function to provide a custom row merge.
/// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting.
/// </summary>
protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right)
protected virtual IBlueRow MergeRows(IBlueRow? left, IBlueRow? right)
{
Dictionary<string, object> newRow = new Dictionary<string, object>();
Dictionary<string, object?> newRow = new();
if (left == null && right == null) throw new ArgumentException("oh come on.");
@@ -393,14 +404,14 @@ namespace BCG
}
}
newRow["id"] = this.myID++;
newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn];
newRow[NEW_TIME_COLUMN] = (left != null) ? left[myLeftTimeColumn] : right![myRightTimeColumn];
return new BlueRow(this.OutputSchema, newRow);
}
private IEnumerable<RowMatch> FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
private IEnumerable<RowMatch>? FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
{
List<RowMatch> matches = null;
List<RowMatch>? matches = null;
// Search through all rows in list and compare with the fuzzy comparer
for (int x = 0; x < list.Count; x++)
@@ -421,21 +432,15 @@ namespace BCG
matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch);
// Mark the best match for this particular right hand match
RowMatch oldMatch;
if (this.myBestMatch.TryGetValue(match, out oldMatch))
{
if (currentMatch.myFitness < oldMatch.myFitness)
{
if (myBestMatch.TryGetValue(match, out var oldMatch)) {
if (currentMatch.myFitness < oldMatch.myFitness) {
this.myBestMatch[match] = currentMatch;
if (oldMatch.myMergedRow != null)
{
if (oldMatch.myMergedRow != null) {
// We already emitted oldMatch.MergedRow so we need to fix it
corrections.Add(oldMatch);
}
}
}
else
{
} else {
this.myBestMatch[match] = currentMatch;
}
}
@@ -7,6 +7,8 @@ using System.Text.Json;
using bts;
using bts.SilverPine;
using mbts.utils;
namespace BCG {
//[DataStreamUserRespresentation("AggregateStream")]
public class NonIncGroupByStream : ViewStream {
@@ -80,7 +82,7 @@ namespace BCG {
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
private void _main_stream_onchange(object nv) {
DataPacket dp = nv as DataPacket;
var dp = ExceptionAssert.Argument.Is<DataPacket>(nv);
///grouping together
_timer.Restart();
IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
@@ -109,7 +111,12 @@ namespace BCG {
}
// TODO,
if (total_state != null)
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
myFireEvent(new DataPacket(
total_state,
total_state,
myLastValue == null
? new BlueTable(total_state.schema)
: ExceptionAssert.of_type<DataPacket>(myLastValue).newRows));
}
}
}
@@ -1,6 +1,5 @@
using System;
using System.Collections;
using BCG.Utilities;
using System.Collections.Generic;
namespace BCG
{
@@ -11,7 +10,8 @@ namespace BCG
/// </summary>
internal class JoinOperandRowIndex
{
private IDictionary myJoinColDict = new Hashtable (), myIdDict = new Hashtable ();
readonly Dictionary<object, object> myJoinColDict = new();
readonly Dictionary<object, object> myIdDict = new();
private int myJoinColOrdinal, myIdColOrdinal;
/// <summary>
@@ -35,18 +35,18 @@ namespace BCG
public void add (IBlueRow row)
{
RowList list;
if (myJoinColDict[row[myJoinColOrdinal]] == null)
myJoinColDict[row[myJoinColOrdinal]] = list = new RowList ();
if (myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value] == null)
myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value] = list = new RowList ();
else
list = (RowList)myJoinColDict[row[myJoinColOrdinal]];
myIdDict[row[myIdColOrdinal]] = list.add (row);
list = (RowList)myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value];
myIdDict[row[myIdColOrdinal] ?? DBNull.Value] = list.add (row);
}
public void remove (IBlueRow row)
{
RowNode node = (RowNode)myIdDict[row[myIdColOrdinal]];
((RowList)myJoinColDict[row[myJoinColOrdinal]]).remove (node);
myIdDict.Remove (row[myIdColOrdinal]);
RowNode node = (RowNode)myIdDict[row[myIdColOrdinal] ?? DBNull.Value];
((RowList)myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value]).remove (node);
myIdDict.Remove (row[myIdColOrdinal] ?? DBNull.Value);
}
public RowList getMatchingRows (object key)
@@ -1,18 +1,20 @@
using System;
using BCG.Utilities;
using System.Collections;
using bts;
using Bts.DataStreamCore;
namespace BCG
{
internal class JoinRow : IBlueRow
{
private RowNode myLeftNode, myRightNode;
protected IBlueRow myLeftRow, myRightRow;
RowNode? myLeftNode;
RowNode? myRightNode;
protected IBlueRow myLeftRow;
protected IBlueRow? myRightRow;
protected int myId;
protected JoinTableSchema mySchema;
private object[] myValues;
private object?[] myValues;
#region Constructor(s)
/// <summary>
@@ -21,8 +23,8 @@ namespace BCG
/// <param name="leftRow">left row of the join</param>
/// <param name="rightRow">right row of the join</param>
/// <param name="schema">schema of the joined row</param>
public JoinRow (IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
: this ((int)leftRow["id"], leftRow, rightRow, schema) {}
public JoinRow(IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
: this(leftRow.Id(), leftRow, rightRow, schema) { }
/// <summary>
/// Constructs a row assigning it the ID provioded as the first argument
@@ -31,7 +33,7 @@ namespace BCG
/// <param name="leftRow">left row of the join</param>
/// <param name="rightRow">right row of the join</param>
/// <param name="schema">schema of the joined row</param>
public JoinRow (int id, IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
public JoinRow (int id, IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema)
{
myId = id;
myLeftRow = leftRow;
@@ -48,13 +50,13 @@ namespace BCG
public bool isPartialJoin{get{return myRightRow == null;}}
public RowNode leftNode
public RowNode? leftNode
{
get{return myLeftNode;}
set{myLeftNode = value;}
}
public RowNode rightNode
public RowNode? rightNode
{
get{return myRightNode;}
set{myRightNode = value;}
@@ -85,7 +87,7 @@ namespace BCG
/// The key can either be the string corrsponding to the column name or
/// the order of the column
/// </summary>
public virtual object this [object key]
public virtual object? this [object key]
{
get
{
@@ -96,7 +98,7 @@ namespace BCG
set {throw new NotSupportedException ("The IBlueRow is read only");}
}
public object this [int index]
public object? this [int index]
{
get
{
@@ -119,7 +121,7 @@ namespace BCG
}
}
public object this [string colName] {get{return this[mySchema[colName].colNo];}}
public object? this [string colName] {get{return this[mySchema[colName].colNo];}}
public ICollection Keys{get{return mySchema.columnNames;}}
@@ -133,9 +135,11 @@ namespace BCG
}
}
public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
public void Add(object key, object? value) =>
throw new NotSupportedException("The IBlueRow is readonly");
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
public void Clear() =>
throw new NotSupportedException ("The IBlueRow is readonly");
public bool Contains (object key){return mySchema[(string)key] != null;}
@@ -1,11 +1,10 @@
using System;
using System.Data;
using System.Collections;
using System.Text.RegularExpressions;
using BCG.Utilities;
using BCG.Utilities.General;
using bts;
using mbts.utils;
using System;
using System.Collections;
namespace BCG
{
public class JoinStream : ViewStream
@@ -111,8 +110,15 @@ namespace BCG
{
if (myPs == null)
{
JoinNameResolver resolver = new JoinNameResolver ((JoinTableSchema)myIjs.schema, myIjs.getSchema (true), myLeftCm.columnExpression,
myIjs.getSchema (false), myRightCm.columnExpression);
ArgumentNullException.ThrowIfNull(myIjs);
ArgumentNullException.ThrowIfNull(myLeftCm);
ArgumentNullException.ThrowIfNull(myRightCm);
var resolver = new JoinNameResolver(
ExceptionAssert.of_type<JoinTableSchema>(myIjs.schema),
myIjs.getSchema(true),
myLeftCm.columnExpression,
myIjs.getSchema(false),
myRightCm.columnExpression);
IDictionary map = resolver.getMap ();
map["id"] = "id";
myPs = new ProjectStream (myIjs, map);
@@ -123,26 +129,27 @@ namespace BCG
Log.assert(false, "This call back should not have been called");
}
else
myFireEvent ((o as ICloneable).Clone());
myFireEvent(ExceptionAssert.of_type<ICloneable>(o).Clone());
}
private void myProjectChangeHandler (object o)
{
ArgumentNullException.ThrowIfNull(myIjs);
if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1)
myIjs.onChange -= myInternalJoinChangeHandler;
myFireEvent ((o as ICloneable).Clone());
myFireEvent(ExceptionAssert.of_type<ICloneable>(o).Clone());
}
protected override void myOnFirstSubscriber ()
{
myIjs.onChange += myInternalJoinChangeHandler;
ExceptionAssert.not_null(myIjs).onChange += myInternalJoinChangeHandler;
}
protected override void myOnLastSubscriber ()
{
// how does this relate to the -= in myProjectChangeHandler?
myIjs.onChange -= myInternalJoinChangeHandler;
ExceptionAssert.not_null(myIjs).onChange -= myInternalJoinChangeHandler;
// if there is a project after the ijs we need to
// unsubscribe to it otherwise references will be kept and
@@ -159,7 +166,9 @@ namespace BCG
/// </summary>
/// <param name="leftPacket"></param>
/// <param name="rightPacket"></param>
public void test (DataPacket dp, bool isLeft) {myIjs.test (dp, isLeft);}
public void test (DataPacket dp, bool isLeft) =>
ExceptionAssert.not_null(myIjs).test(dp, isLeft);
/***********************************************************************************************/
}
}
@@ -1,7 +1,7 @@
using Bts.DataStreamCore;
using System;
using System.Collections;
using System.IO;
using BCG.Utilities.General;
namespace BCG
{
@@ -19,20 +19,13 @@ namespace BCG
myRightIndex = new JoinDeletionIndex (mySchema.getRenamedName ("id", false), false);
}
public IBlueRow this [int id]
{
get
{
if (myIdIndex[id] == null)
throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
return (IBlueRow)myIdIndex[id];
}
}
public IBlueRow this [int id] =>
myIdIndex[id] as IBlueRow ?? throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
#region JoinTable members
public virtual void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp)
public virtual void addRow (IBlueRow leftRow, IBlueRow? rightRow, DataPacket dp)
{
JoinRow newRow = new JoinRow (myNewRowId, leftRow, rightRow, mySchema);
var newRow = new JoinRow(myNewRowId, leftRow, rightRow, mySchema);
myIdIndex[myNewRowId ++] = newRow;
((BlueTable)(dp.addedRows)).addRow (newRow);
myLeftIndex.add (newRow);
@@ -48,17 +41,17 @@ namespace BCG
thisIndex = myLeftIndex;
otherIndex = myRightIndex;
}
RowList matchingNodes = thisIndex.getMatchingRows (deletedRow["id"]);
RowList matchingNodes = thisIndex.getMatchingRows (deletedRow.Id());
foreach (RowNode node in matchingNodes)
{
JoinRow row = (JoinRow)node.row;
RowNode correspondingNode = isLeft ? row.rightNode : row.leftNode;
var correspondingNode = isLeft ? row.rightNode : row.leftNode;
if (correspondingNode != null)
otherIndex.remove (correspondingNode); //in case of partial left joinss
myIdIndex.Remove (row["id"]);
myIdIndex.Remove (row.Id());
((BlueTable)dp.deletedRows).addRow (row);
}
thisIndex.removeAll (deletedRow["id"]);
thisIndex.removeAll(deletedRow.Id());
}
#endregion
#region ICollection members
@@ -1,9 +1,10 @@
using System;
using System.Collections;
using BCG.Utilities;
using System.Collections.Generic;
using System.Linq;
using bts;
using mbts.utils;
namespace BCG
{
/// <summary>
@@ -13,10 +14,11 @@ namespace BCG
public class JoinTableSchema : BlueTableSchema
{
private string myLeftJc = "", myRightJc = "", myLeftId = "", myRightId = "";
private IDictionary myLeftNameToAlias = new Hashtable ();
private IDictionary myRightNameToAlias = new Hashtable ();
private IDictionary myAliasToLeftName = new Hashtable(),
myAliasToRightName = new Hashtable();
readonly Dictionary<string, string> myLeftNameToAlias = new();
readonly Dictionary<string, string> myRightNameToAlias = new();
readonly Dictionary<string, string> myAliasToLeftName = new();
readonly Dictionary<string, string> myAliasToRightName = new();
/// <summary>
/// Constructs the schema of the Jointable with the schema of the left and right tables. Incase of
/// a name being there in both the shemas it prefixes the name with "L." or "R." accordingly. Column "id"
@@ -85,9 +87,9 @@ namespace BCG
renamed = false;
}
myIndexByColName["id"] = new ColumnInfo("id", myIndexByColName.Count, typeof(int));
myIndexByColOrder[myIndexByColOrder.Length - 1] = (ColumnInfo)myIndexByColName["id"];
myAliasToLeftName = ProjectHelper.invertDic(myLeftNameToAlias);
myAliasToRightName = ProjectHelper.invertDic(myRightNameToAlias);
myIndexByColOrder[myIndexByColOrder.Length - 1] = ExceptionAssert.of_type<ColumnInfo>(myIndexByColName["id"]);
myAliasToLeftName = myLeftNameToAlias.ToDictionary(kv => kv.Value, kv => kv.Key);
myAliasToRightName = myRightNameToAlias.ToDictionary(kv => kv.Key, kv => kv.Value);
}
public string leftJoinCol {get{return myLeftJc;}}
@@ -97,8 +99,8 @@ namespace BCG
public string getRenamedName (string origName, bool isLeft)
{
if (isLeft)
return (string)myLeftNameToAlias[origName];
return (string)myRightNameToAlias[origName];
return myLeftNameToAlias[origName] as string ?? string.Empty;
return myRightNameToAlias[origName] as string ?? string.Empty;
}
public string getOriginalName(string rename, bool isLeft)
@@ -2,16 +2,15 @@
using bts.collections;
using mbts;
using mbts.utils;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Security.Cryptography;
namespace BCG {
public class NonIncJoinBookkeeper : IDisposable {
public Action on_new_data;
public Action? on_new_data;
private readonly NonIncJoinSpec _join_spec;
private readonly int _join_position;
private readonly int _start_index_in_output;
@@ -20,16 +19,19 @@ namespace BCG {
private readonly bool _is_tag_along;
private int _n_total_fires = 0, _n_expensive_fires = 0;
private IBlueTableSchema _schema = null;
private IBlueTableSchema? _schema;
private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
private IBlueRow _solitary_row = null;
private IBlueRow? _solitary_row;
private int[] _join_col_indices = null,
_data_col_indices = null;
int[]? _join_col_indices;
int[]? _data_col_indices;
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
_join_spec = the_join_spec;
public NonIncJoinBookkeeper(NonIncJoinSpec join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
if (join_spec.join_col_names is null) {
throw new ArgumentException("join_col_names is null", nameof(join_spec));
}
_join_spec = join_spec;
_join_position = join_position;
_start_index_in_output = start_index_in_output;
_row_hasher = row_hasher;
@@ -41,7 +43,7 @@ namespace BCG {
private void _fill_missing_data(DataPacket dp) {
if (_filled_missing_data)
return;
Func<string[], bool> null_or_empty = (str_arr) => {
bool null_or_empty(string[]? str_arr) {
if (str_arr == null || str_arr.Length == 0)
return true;
foreach (string s in str_arr) {
@@ -50,13 +52,16 @@ namespace BCG {
}
}
return true;
};
Func<string[], string[]> remove_bad_cols = (col_names) => {
}
string[] remove_bad_cols(string[]? col_names) {
if (col_names is null) {
return Array.Empty<string>();
}
HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
List<string> cols_list = new List<string>(col_names);
cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
return cols_list.ToArray();
};
}
if (null_or_empty(_join_spec.data_col_names)) {
_join_spec.data_col_names = dp.schema.columnNames;
}
@@ -70,7 +75,7 @@ namespace BCG {
}
public void init() {
_join_spec.stream.onChange += _on_stream_cb;
ExceptionAssert.not_null(_join_spec.stream).onChange += _on_stream_cb;
}
public void _on_stream_cb(object dp_obj) {
@@ -111,14 +116,19 @@ namespace BCG {
public bool is_ready { get { return _schema != null; } }
public void constribute_cols_to_schema(List<ColumnInfo> cis) {
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
ArgumentNullException.ThrowIfNull(_join_spec.data_col_names);
if (_join_spec.data_col_names.Length > 0) {
ArgumentNullException.ThrowIfNull(_join_spec.projected_col_names);
ArgumentNullException.ThrowIfNull(_schema);
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
}
}
public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
public void copy_master_row_to_output_row(int row_index, object[] output_row) {
IBlueRow this_row = _row_list[row_index].second;
IBlueRow? this_row = _row_list[row_index].second;
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
}
@@ -129,8 +139,7 @@ namespace BCG {
else
copy_dbnull_to_output_row(output_row);
} else {
IBlueRow this_row = null;
if (_row_map.TryGetValue(row_hash, out this_row))
if (_row_map.TryGetValue(row_hash, out var this_row))
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
else
copy_dbnull_to_output_row(output_row);
@@ -138,13 +147,16 @@ namespace BCG {
}
private void copy_dbnull_to_output_row(object[] output_row) {
ArgumentNullException.ThrowIfNull(_data_col_indices);
for (int i = 0; i < _data_col_indices.Length; i++)
output_row[i + _start_index_in_output] = DBNull.Value;
}
#region IDisposable
public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
public void Dispose() {
ExceptionAssert.not_null(_join_spec.stream).onChange -= _on_stream_cb;
}
#endregion
}
@@ -1,20 +1,18 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using mbts;
using mbts.utils;
namespace BCG {
// this a collection of data, theres no joining logic in here
public class NonIncJoinSpec {
public ViewStream stream;
public ViewStream? stream;
// join col names are the names on which the streams are joined (for equality),
// data col names are the names in the original stream
// projected col names are their new names, ie if its x:l_x, then x is in the
// data_col_names and l_x in projected cols
public string[] join_col_names, data_col_names, projected_col_names;
public string[]? join_col_names, data_col_names, projected_col_names;
public NonIncJoinSpec() { }
@@ -31,8 +29,8 @@ namespace BCG {
public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
foreach (Triplet<ViewStream, string, string> r in join_specs) {
string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
string[] visible_cols = r.third.Split(' ');
string[] keys = (r.second ?? string.Empty).Split(' ');
string[] visible_cols = ExceptionAssert.not_null(r.third).Split(' ');
List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
foreach (string s in visible_cols) {
string[] toks = s.Split(':');
@@ -44,22 +42,15 @@ namespace BCG {
mapped_col_names.Add(toks[1]);
}
}
rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
rv.Add(new NonIncJoinSpec(ExceptionAssert.not_null(r.first), keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
}
return rv.ToArray();
}
public override string ToString() {
Func<string[], string> arr_txt = (a) => {
string rv = "";
foreach (string s in a) {
rv += "{s} ";
}
rv = rv.Trim();
return rv;
};
string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
return rv;
}
public override string ToString() =>
$"stream: {stream} "
+ $"join_col_names: { string.Join(" ", join_col_names ?? Array.Empty<string>())} "
+ $"data_col_names: { string.Join(" ", data_col_names ?? Array.Empty<string>())} "
+ $"projected_col_names: { string.Join(" ", projected_col_names ?? Array.Empty<string>())}";
}
}
@@ -1,7 +1,14 @@
using System;
using System.Collections;
using System.Collections.Generic;
using bts;
using Bts.DataStreamCore;
using mbts;
using mbts.utils;
using Spark.Core.Collections;
namespace BCG
@@ -178,10 +185,11 @@ namespace BCG
public class SuperJoinRow : IBlueRow
{
private JoinProjectHelper myHelper;
private IBlueRow myLeftRow, myRightRow;
private IBlueRow myLeftRow;
private IBlueRow? myRightRow;
int myID;
public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow rightRow, JoinProjectHelper joinProjectHelper)
public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow? rightRow, JoinProjectHelper joinProjectHelper)
{
myID = id;
myLeftRow = leftRow;
@@ -189,21 +197,21 @@ namespace BCG
myHelper = joinProjectHelper;
}
public int leftID { get { return (int)myLeftRow["id"]; } }
public int leftID => myLeftRow.Id();
public int rightID { get { return (int)myRightRow["id"]; } }
public int rightID => ExceptionAssert.not_null(myRightRow).Id();
public IBlueRow leftRow { get { return myLeftRow; } }
public IBlueRow rightRow { get { return myRightRow; } }
public IBlueRow? rightRow => myRightRow;
#region IBlueRow Members
public IBlueTableSchema schema { get { return myHelper.schema; } }
public object this[int index] { get { return this[schema[index].colName]; } }
public object? this[int index] { get { return this[schema[index].colName]; } }
public object this[string colName]
public object? this[string colName]
{
get
{
@@ -211,8 +219,7 @@ namespace BCG
return myID;
else
{
bool isLeft;
string col = myHelper.getOriginalCol(colName, out isLeft);
string col = myHelper.getOriginalCol(colName, out bool isLeft);
return isLeft ? myLeftRow[col] : (myRightRow == null ? (object)DBNull.Value : myRightRow[col]);
}
}
@@ -229,7 +236,7 @@ namespace BCG
public bool Contains(object key)
{
return myLeftRow.Contains(key) || myRightRow.Contains(key);
return myLeftRow.Contains(key) || (myRightRow?.Contains(key) ?? false);
}
public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
@@ -246,7 +253,7 @@ namespace BCG
{
get
{
List<object> rv = new List<object>();
var rv = new List<object?>();
foreach (string col in Keys)
rv.Add(this[col]);
return rv;
@@ -291,9 +298,10 @@ namespace BCG
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
private bool myManageUnmappedLeftRows = false;
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
protected IBlueTableSchema myLeftSchema, myRightSchema;
protected IBlueTableSchema? myLeftSchema;
protected IBlueTableSchema? myRightSchema;
protected JoinProjectHelper myJoinProjectHelper;
protected BlueTable myState;
protected BlueTable? myState;
protected int myID;
protected ViewStream myLeftStream, myRightStream;
protected string myLeftKeyCol, myRightKeyCol;
@@ -346,25 +354,25 @@ namespace BCG
private void myHandleLeftStreamChanges(object newValue)
{
myHandleStreamChanges(newValue as DataPacket, true);
myHandleStreamChanges(ExceptionAssert.Argument.Is<DataPacket>(newValue), true);
}
private void myHandleRightStreamChanges(object newValue)
{
myHandleStreamChanges(newValue as DataPacket, false);
myHandleStreamChanges(ExceptionAssert.Argument.Is<DataPacket>(newValue), false);
}
private void myHandleStreamChanges(DataPacket dp, bool isLeft)
{
myInitSchema(dp.schema, isLeft);
BlueTable dels = null, adds = null;
BlueTable? dels = null, adds = null;
if (myState != null)
{
dels = new BlueTable(myState.schema);
adds = new BlueTable(myState.schema);
}
myDeleteRows(dp.deletedRows, isLeft, myState, dels);
myAddRows(dp.addedRows, isLeft, myState, adds);
myDeleteRows(dp.deletedRows, isLeft, ExceptionAssert.not_null(myState), ExceptionAssert.not_null(dels));
myAddRows(dp.addedRows, isLeft, myState, ExceptionAssert.not_null(adds));
if (myState != null)
{
myFireEvent(new DataPacket(myState, adds, dels));
@@ -410,8 +418,8 @@ namespace BCG
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
foreach (IBlueRow r in addedRows)
{
object currentKey = r[thisKeyCol];
int currentID = (int)r["id"];
var currentKey = r[thisKeyCol] ?? DBNull.Value;
int currentID = r.Id();
///add it to 1.
if (!thisKeyToRows.ContainsKey(currentKey))
thisKeyToRows.Add(currentKey, new Dictionary<int, IBlueRow>());
@@ -422,14 +430,14 @@ namespace BCG
ICollection<IBlueRow> matchingRows = otherSideKeyToRows[currentKey].Values;
foreach (IBlueRow mr in matchingRows)
{
int mrID = (int)mr["id"];
int mrID = mr.Id();
SuperJoinRow rr = new SuperJoinRow(myID++, isLeft ? r : mr, isLeft ? mr : r, myJoinProjectHelper);
state.addRow(rr);
addedRowsOut.addRow(rr);
///add to 3.
if (!thisIDToResultRows.ContainsKey(currentID))
thisIDToResultRows.Add(currentID, new Dictionary<int, SuperJoinRow>());
thisIDToResultRows[currentID].Add((int)rr["id"], rr);
thisIDToResultRows[currentID].Add(rr.Id(), rr);
///add to 4.
if (!otherIDToResultRows.ContainsKey(mrID))
{
@@ -437,7 +445,7 @@ namespace BCG
if (myManageUnmappedLeftRows && !isLeft)
myTakeCareOfUnmatachedLeftRows(mr, false);
}
otherIDToResultRows[mrID].Add((int)rr["id"], rr);
otherIDToResultRows[mrID].Add(rr.Id(), rr);
}
}
else
@@ -458,22 +466,21 @@ namespace BCG
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
foreach (IBlueRow r in deletedRows)
{
int id = (int)r["id"];
object key = r[thisKeyCol];
int id = r.Id();
object key = ExceptionAssert.not_null(r[thisKeyCol]);
///delete it from 1.
thisKeyToRows[key].Remove(id);
if (thisKeyToRows[key].Count == 0)
thisKeyToRows.Remove(key);
///form return rows and delete from 3.
if (thisIDToResultRows.ContainsKey(id))
if (thisIDToResultRows.TryGetValue(id, out IDictionary<int, SuperJoinRow>? removedResultRows))
{
ICollection removedResultRows = thisIDToResultRows[id].Values as ICollection;
thisIDToResultRows.Remove(id);
foreach (SuperJoinRow rr in removedResultRows)
foreach (var rr in removedResultRows.Values)
{
///delete from 4.
int otherId = isLeft ? rr.rightID : rr.leftID;
otherIDToResultRows[otherId].Remove((int)rr["id"]);
otherIDToResultRows[otherId].Remove(rr.Id());
if (otherIDToResultRows[otherId].Count == 0)
otherIDToResultRows.Remove(otherId);
delRowsOut.addRow(rr);
@@ -493,15 +500,11 @@ namespace BCG
private void myTakeCareOfUnmatachedLeftRows(IBlueRow leftRow, bool add)
{
int lid = (int)leftRow["id"];
if (add)
int lid = leftRow.Id();
if (add) {
myLeftUnmatchedRowsAdds.Add(lid, leftRow);
else
{
if (myLeftUnmatchedRowsAdds.ContainsKey(lid))
myLeftUnmatchedRowsAdds.Remove(lid);
else
myLeftUnmatchedRowsDels.Add((int)leftRow["id"], leftRow);
} else if (!myLeftUnmatchedRowsAdds.Remove(lid)) {
myLeftUnmatchedRowsDels.Add(lid, leftRow);
}
}
}
@@ -522,44 +525,48 @@ namespace BCG
protected override void myFireEvent(object o)
{
myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, o as DataPacket);
var dp = ExceptionAssert.Argument.Is<DataPacket>(o);
myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, dp);
}
private void myHandleJoinResults(Dictionary<int, IBlueRow> unmatchedLeftRowsDels,
Dictionary<int, IBlueRow> unmatchedLeftRowAdds, DataPacket resultPacket)
{
BlueTable dels = resultPacket.deletedRows as BlueTable,
adds = resultPacket.addedRows as BlueTable;
foreach (IBlueRow lr in unmatchedLeftRowsDels.Values)
{
///an lr will be here if its actually deleted or
///a right row is added, so now it has a partner
///and is no longer unmatched.
///in both cases the half row that this lr
///created needs to be removed from this
///class's state and added to dels
///it means a right is added
int lid = (int)lr["id"];
SuperJoinRow rr = myLeftIDToHalfRows[lid];
myLeftIDToHalfRows.Remove(lid);
myState.deleteRow(rr);
dels.addRow(rr);
var dels = ExceptionAssert.of_type<BlueTable>(resultPacket.deletedRows);
var adds = ExceptionAssert.of_type<BlueTable>(resultPacket.addedRows);
if (unmatchedLeftRowsDels.Count > 0) {
Log.notnull(myState);
foreach (IBlueRow lr in unmatchedLeftRowsDels.Values) {
///an lr will be here if its actually deleted or
///a right row is added, so now it has a partner
///and is no longer unmatched.
///in both cases the half row that this lr
///created needs to be removed from this
///class's state and added to dels
///it means a right is added
int lid = lr.Id();
SuperJoinRow rr = myLeftIDToHalfRows[lid];
myLeftIDToHalfRows.Remove(lid);
myState.deleteRow(rr);
dels.addRow(rr);
}
}
foreach (IBlueRow lr in unmatchedLeftRowAdds.Values)
{
///an lr will be here if its added with no
///matching right, or a right matching row
///is gone so it is unmatched now.
///in both cases a half row needs to be
///added to this class's state and added
///to adds
int lid = (int)lr["id"];
SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
myLeftIDToHalfRows.Add(lid, rr);
myState.addRow(rr);
adds.addRow(rr);
if (unmatchedLeftRowAdds.Count > 0) {
Log.notnull(myState);
foreach (IBlueRow lr in unmatchedLeftRowAdds.Values) {
///an lr will be here if its added with no
///matching right, or a right matching row
///is gone so it is unmatched now.
///in both cases a half row needs to be
///added to this class's state and added
///to adds
int lid = lr.Id();
var rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
myLeftIDToHalfRows.Add(lid, rr);
myState.addRow(rr);
adds.addRow(rr);
}
}
base.myFireEvent(new DataPacket(myState, adds, dels));
}
@@ -1,6 +1,9 @@
using bts;
using Bts.DataStreamCore;
using System;
using System.Collections;
using BCG.Utilities;
namespace BCG
{
@@ -11,9 +14,10 @@ namespace BCG
[DataStreamUserRespresentation (false)]
public class LeftInternalJoinStream : InternalJoinStream
{
protected DataPacket myLeftDataPacket, myRightDataPacket;
protected DataPacket? myLeftDataPacket, myRightDataPacket;
//Container to contain all the left rows that are orphaned during a deletion operation
private IDictionary myLeftOrphanedRows;
IDictionary? myLeftOrphanedRows;
public LeftInternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight) :
base (vsLeft, jcLeft, vsRight, jcRight) {}
@@ -62,10 +66,10 @@ namespace BCG
private void myTestAndFire (bool isLeft)
{
JoinOperand jo = isLeft ? myLeftOperand : myRightOperand;
DataPacket dp = isLeft ? myLeftDataPacket : myRightDataPacket;
var dp = isLeft ? myLeftDataPacket : myRightDataPacket;
//to test whether both have received call backs
if (myLeftOperand.hasReceivedData)
jo.changeHandler (dp);
jo.changeHandler(Log.notnull(dp));
else if (myLeftDataPacket != null && myRightDataPacket != null)
{
myRightOperand.changeHandler (myRightDataPacket);
@@ -82,9 +86,11 @@ namespace BCG
{
foreach (IBlueRow row in rows)
myAddRow (row, isLeft);
if (!isLeft)
if (!isLeft) {
Log.notnull(myLeftOrphanedRows);
foreach (RowNode node in myLeftOrphanedRows.Values)
((JoinTable)myLastFiredPacket.newRows).addRow (node.row, null, myLastFiredPacket);
((JoinTable)myLastFiredPacket.newRows).addRow(node.row, null, myLastFiredPacket);
}
}
/// <summary>
@@ -104,7 +110,7 @@ namespace BCG
if (isLeft)
{
myLeftOperand.addRow (row);
RowList matchedRows = myRightOperand.getMatchingRows (row[myLeftJoinColOrdinal]);
RowList matchedRows = myRightOperand.getMatchingRows (row[myLeftJoinColOrdinal] ?? DBNull.Value);
if (matchedRows.Count == 0)
((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
else
@@ -113,10 +119,10 @@ namespace BCG
}
else
{
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]);
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value);
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count;
myRightOperand.addRow (row);
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count;
if (matchedRows.Count != 0){
foreach (RowNode matchedNode in matchedRows)
{
@@ -125,10 +131,11 @@ namespace BCG
//the jointable as a new row will replace it
if (rightMatchingRowCount == 0 && newRightMatchingRowCount > 0)
{
if (myLeftOrphanedRows[matchedNode.row["id"]] == null)
Log.notnull(myLeftOrphanedRows);
if (myLeftOrphanedRows[matchedNode.row.Id()] == null)
((JoinTable)myLastFiredPacket.newRows).deleteRow (true, matchedNode.row, myLastFiredPacket);
else
myLeftOrphanedRows.Remove (matchedNode.row["id"]);
myLeftOrphanedRows.Remove (matchedNode.row.Id());
}
//they were orphaned by a previous transaction and hence the orphans need to be
//removed from the Jointable and the complete row added
@@ -148,21 +155,23 @@ namespace BCG
}
else
{
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]);
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value);
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count;
myRightOperand.deleteRow (row);
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count;
if (matchedRows.Count != 0)
{
if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0)
if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0) {
//ie all the matchedRows were joined with 'this' right row (method arg), and hence they need to
//be handled specially, ie joined with nulls, if there are no right rows with this
//joincolumn value being inserted.
Log.notnull(myLeftOrphanedRows);
foreach (RowNode matchedNode in matchedRows)
{
// BlueEventLog.logAssert (myLeftOrphanedRows[matchedNode.row["id"]] == null);
myLeftOrphanedRows[matchedNode.row["id"]] = matchedNode;
myLeftOrphanedRows[matchedNode.row.Id()] = matchedNode;
}
}
((JoinTable)myLastFiredPacket.newRows).deleteRow (false, row, myLastFiredPacket);
}
}
@@ -173,9 +182,11 @@ namespace BCG
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
myLastFiredPacket.newRows = new JoinTable (mySchema);
myLastFiredPacket.addedRows = new BlueTable (mySchema);
if (!isLeft)
if (!isLeft) {
Log.notnull(myLeftDataPacket);
foreach (IBlueRow row in myLeftDataPacket.newRows)
((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
((JoinTable)myLastFiredPacket.newRows).addRow(row, null, myLastFiredPacket);
}
}
}
@@ -30,7 +30,7 @@ namespace BCG
/// <summary>
/// Node containing the referencedRow
/// </summary>
IBlueRow referencedRow {get;}
IBlueRow? referencedRow {get;}
/// <summary>
/// removes all referenced or referencing nodes (indicated by the bool), p.s. theres only one referenced row
@@ -1,5 +1,3 @@
using System;
namespace BCG
{
/// <summary>
@@ -15,12 +13,12 @@ namespace BCG
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
IBlueRow getRow (int id);
IBlueRow? getRow (int id);
/// <summary>
/// Gets all the rows in the node
/// </summary>
/// <returns></returns>
IBlueRow[] getRows ();
IBlueRow?[] getRows ();
/// <summary>
/// Replaces the row with the given id, with the row passed as the second arg
/// </summary>
@@ -1,7 +1,7 @@
using Bts.DataStreamCore;
using System;
using System.Collections;
using BCG.Utilities;
using BCG;
namespace BCG
{
@@ -56,18 +56,18 @@ namespace BCG
/// The key can either be the string corrsponding to the column name or
/// the order of the column
/// </summary>
public virtual object this [object key]
public virtual object? this [object key]
{
get {return key is string ? this [key as string] : this [(int)key];}
get => key is string ks ? this[ks] : this[(int)key];
set {throw new NotSupportedException ("The IBlueRow is read only");}
}
public virtual object this [int index]
public virtual object? this [int index]
{
get
{
if (index == mySchema.columnNames.Length - 1)
return myRingGRow["id"];
return myRingGRow.Id();
if (index < myRingGRow.Count)
return myRingGRow[index];
else if (index >= myRingGRow.Count && index < mySchema.columnNames.Length - 1)
@@ -77,7 +77,8 @@ namespace BCG
}
}
public object this [string colName] {get{return this[mySchema[colName].colNo];}}
public object? this [string colName] =>
this[mySchema[colName].colNo];
public ICollection Keys{get{return mySchema.columnNames;}}
@@ -1,6 +1,7 @@
using mbts.utils;
using System;
using System.Collections;
using BCG.Utilities;
namespace BCG
{
@@ -70,7 +71,10 @@ namespace BCG
}
else
{
SimpleNode sn = n as SimpleNode;
var sn = ExceptionAssert.Argument.Is<SimpleNode>(n);
if (sn.prev is null) {
throw new ArgumentException("prev is null", nameof(n));
}
sn.prev.next = sn.next;
if (sn.next != null)
sn.next.prev = sn.prev;
@@ -81,7 +85,7 @@ namespace BCG
public int count {get {return myFirst.listCount;}}
public IBlueRow referencedRow {get {return myFirst.row;}}
public IBlueRow? referencedRow => myFirst.row;
public void removeAll (bool isReD)
{
@@ -110,10 +114,17 @@ namespace BCG
public class LightForeignJoinRowListEnumerator : IEnumerator
{
private ILightForeignJoinRowList myList;
private SimpleNode myPtr, myFirst;
public LightForeignJoinRowListEnumerator (ILightForeignJoinRowList list, SimpleNode first)
{myList = list; myFirst = first;}
public object Current {get{return myPtr.row;}}
private SimpleNode? myPtr;
private SimpleNode myFirst;
public LightForeignJoinRowListEnumerator(ILightForeignJoinRowList list, SimpleNode first) {
myList = list;
myFirst = first;
}
public object? Current =>
myPtr is null ? throw new InvalidOperationException("myPtr is null") : myPtr.row;
public bool MoveNext() {return (myPtr = myPtr == null ? myFirst : myPtr.next) != null;}
public void Reset() {myPtr = null;}
public ILightForeignJoinRowList list {get {return myList;}}
@@ -1,5 +1,6 @@
using Bts.DataStreamCore;
using System;
using BCG.Utilities;
namespace BCG
{
@@ -13,12 +14,12 @@ namespace BCG
: base (reDRow, rinGRow, schema)
{}
public override object this [int index]
public override object? this [int index]
{
get
{
if (index == mySchema.columnNames.Length - 1)
return myRingGRow["id"];
return myRingGRow.Id();
if (index < myRingGRow.Count)
return myRingGRow[index];
else if (index >= myRingGRow.Count && index < mySchema.columnNames.Length - 1)
@@ -1,5 +1,3 @@
using System;
namespace BCG
{
/// <summary>
@@ -9,15 +7,15 @@ namespace BCG
/// </summary>
public class SimpleNode : INode
{
public IBlueRow row;
public SimpleNode next, prev;
public IBlueRow? row;
public SimpleNode? next;
public SimpleNode? prev;
public SimpleNode (IBlueRow r)
: this (r, null, null)
{}
public SimpleNode (IBlueRow r, SimpleNode nx, SimpleNode pv)
{
public SimpleNode(IBlueRow r, SimpleNode? nx, SimpleNode? pv) {
row = r;
next = nx;
prev = pv;
@@ -25,13 +23,15 @@ namespace BCG
#region INode Members
public IBlueRow getRow (int id) {return row;}
public IBlueRow? getRow (int id) =>
row;
public IBlueRow[] getRows() {return new IBlueRow[] {row};}
public IBlueRow?[] getRows() =>
new[] { row };
public void replace (int id, IBlueRow nr) {row = nr;}
#endregion
}
}
}
@@ -1,14 +1,8 @@
using System;
using System.Collections;
using System.Collections.Generic;
using Amazon.Runtime.Internal.Util;
using BCG;
using BCG.Utilities;
using bts;
using System;
using System.Collections.Generic;
namespace BCG {
// Prefixing with "old" to avoid name conflict with the BlueOrb LiteralStream
[DataStreamUserRespresentation("oldLiteral")]
@@ -44,13 +38,14 @@ namespace BCG {
return new BlueRow(map);
};
string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
BlueTable table = null;
BlueTable? table = null;
foreach (string json_string in row_arr) {
IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
get_row(json_string, table == null ? 0 : table.Count);
table = table ?? new BlueTable(r.schema);
table ??= new BlueTable(r.schema);
table.addRow(r);
}
table ??= BlueTable.Empty(table_name);
ViewStream str = new BlueTableStream(table);
str = new SchemaGuesserStream(str);
mySetRealSource(str);
@@ -67,8 +62,11 @@ namespace BCG {
string[] columnAndType = c.Split('/');
if (columnAndType.Length == 1)
ci[n] = new ColumnInfo(c, n, typeof(double));
else
ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
else {
var colType = Type.GetType("System." + columnAndType[1], true)
?? throw new ArgumentException($"Failed to get column type at index {n}", nameof(schema));
ci[n] = new ColumnInfo(columnAndType[0], n, colType);
}
n++;
}
@@ -1,13 +1,14 @@
using bts;
using Bts.ColumnMapExpressions;
using Bts.DataStream;
using mbts.config;
using mbts.utils;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
// using BCG.Evaluator.ColumnMapExpression;
namespace BCG
{
@@ -26,7 +27,7 @@ namespace BCG
/// <param name="isch">original(unprojected) schema</param>
/// <param name="exp">expression defining the mapping</param>
/// <param name="tableName">table name of the new schema</param>
internal ProjectHelper (IBlueTableSchema isch, string exp, string tableName)
internal ProjectHelper (IBlueTableSchema isch, string exp, string? tableName)
: this (isch, myGetDictionary (isch, exp), tableName)
{}
@@ -36,10 +37,10 @@ namespace BCG
/// <param name="origToAlias">dictionary mapping unprojected names to their projected names, dropped cols should not be there</param>
/// <param name="tableName">table name of the new schema</param>
internal ProjectHelper (
IBlueTableSchema isch, IDictionary<string, string> origToAlias, string tableName)
IBlueTableSchema isch, IDictionary<string, string> origToAlias, string? tableName)
{
ArrayList colMapAl = new ArrayList (),
colInfoAl = new ArrayList ();
var colMapAl = new List<int>();
var colInfoAl = new List<ColumnInfo>();
relevantOrigCols = new string[origToAlias.Count-1];
int c = 0;
ArrayList iColNames = new ArrayList (isch.columnNames);
@@ -49,19 +50,15 @@ namespace BCG
if (origToAlias.TryGetValue(colName, out var colAlias))
{
ColumnInfo ci = isch[colName];
colInfoAl.Add (new ColumnInfo (colAlias,
colInfoAl.Count,
ci.colType)
);
colMapAl.Add (ci.colNo);
colInfoAl.Add(new ColumnInfo (colAlias, colInfoAl.Count, ci.colType));
colMapAl.Add(ci.colNo);
if (ci.colName != "id")
relevantOrigCols[c++] = ci.colName;
}
}
schema = new BlueTableSchema (tableName == null ? isch.tableName : tableName,
colInfoAl.ToArray (typeof(ColumnInfo)) as ColumnInfo[]);
colMap = colMapAl.ToArray (typeof(int)) as int[];
schema = new BlueTableSchema(tableName ?? isch.tableName, colInfoAl.ToArray());
colMap = colMapAl.ToArray();
}
/// <summary>
@@ -75,16 +72,11 @@ namespace BCG
{
IDictionary<string, string> origToAlias = new Dictionary<string, string>();
ColumnMapExpression e;
if (Configuration.get<DataStreamConfig>().UseStaticParser)
{
e = ColumnMapExpression.Parse(exp);
}
else
{
var e = Configuration.get<DataStreamConfig>().UseStaticParser
? ColumnMapExpression.Parse(exp)
// TODO (maybe): Actually use non-static parser (unclear what advantages it will have)
e = ColumnMapExpression.Parse(exp);
}
: ColumnMapExpression.Parse(exp);
Log.notnull(e);
if (e.IsInclusive) {
origToAlias = e.Columns.ToDictionary();
origToAlias["id"] = "id";
@@ -104,7 +96,7 @@ namespace BCG
{
IDictionary retVal = new Hashtable (dic.Count);
foreach (DictionaryEntry de in dic)
retVal.Add (de.Value, de.Key);
retVal.Add(ExceptionAssert.not_null(de.Value), de.Key);
return retVal;
}
}
@@ -1,4 +1,8 @@
namespace BCG {
using bts;
using mbts.utils;
using Spark.Core.Collections;
using System.Collections;
@@ -8,10 +12,11 @@ namespace BCG {
[DataStreamUserRespresentation("oldproject")]
public class ProjectStream : ViewStream
{
private ProjectHelper myHelper;
private string myColExp, myTableName;
private ViewStream myVs;
private IDictionary<string, string> myDic;
ProjectHelper? myHelper;
string? myColExp;
string? myTableName;
ViewStream? myVs;
IDictionary<string, string>? myDic;
/// <summary>
/// </summary>
/// <param name="vs">stream to susbsribe to as the input stream</param>
@@ -54,17 +59,17 @@ namespace BCG {
protected override void myOnFirstSubscriber()
{
myVs.onChange += myHandleChange;
Log.notnull(myVs).onChange += myHandleChange;
}
protected override void myOnLastSubscriber()
{
myVs.onChange -= myHandleChange;
Log.notnull(myVs).onChange -= myHandleChange;
}
internal void myHandleChange (object newValue)
{
DataPacket dp = newValue as DataPacket;
var dp = ExceptionAssert.Argument.Is<IDataPacket>(newValue);
if (myHelper != null)
{
IBlueRowCollection dels, adds;
@@ -79,8 +84,8 @@ namespace BCG {
else
{ //first time a callback is received
myHelper = myDic == null ?
new ProjectHelper (dp.schema, myColExp, myTableName) :
new ProjectHelper (dp.schema, myDic, myTableName);
new ProjectHelper (dp.Schema, myColExp ?? string.Empty, myTableName) :
new ProjectHelper (dp.Schema, myDic, myTableName);
IBlueRowCollection temp;
myFireEvent (new DataPacket (temp = new ProjectedTable (dp.newRows, myHelper),
@@ -1,13 +1,10 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using BCG.Utilities;
using System.Diagnostics;
using Microsoft.Data.Analysis;
using Parquet;
using Parquet.Data;
using mbts.utils;
using Microsoft.Data.Analysis;
using System;
using System.Diagnostics;
using System.IO;
namespace BCG {
[DataStreamUserRespresentation("CSVSaver")]
@@ -40,7 +37,7 @@ namespace BCG {
myStream.onChange -= myHandleChanges;
private void myHandleChanges(object newValue) {
IBlueRowCollection totalState = (newValue as DataPacket).newRows;
IBlueRowCollection totalState = ExceptionAssert.Argument.Is<DataPacket>(newValue).newRows;
if (totalState.Count > 0 && myNWrites < myNMaxWrites) {
myNWrites++;
int i = 0;
@@ -62,7 +59,10 @@ namespace BCG {
IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
BlueTable nRows = new BlueTable(myResultSchema);
nRows.addRow(nr);
myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
BlueTable deletedRows = myLastValue is null
? new(myResultSchema)
: new(ExceptionAssert.of_type<DataPacket>(myLastValue).newRows);
myFireEvent(new DataPacket(nRows, nRows, deletedRows));
} else
myFireEvent(newValue);
}
@@ -83,7 +83,7 @@ namespace BCG {
if (col != "id") {
string colText = String.Empty;
if (r[col] != null && r[col] != DBNull.Value)
colText = r[col].ToString();
colText = r[col]?.ToString() ?? string.Empty;
if (r.schema[col].colType == typeof(string))
colText = "\"" + colText + "\"";
rv += (i++ == 0 ? colText : "," + colText);
@@ -136,7 +136,7 @@ namespace BCG {
}
public class CSVReaderStreamMapper : IStreamMapper {
private IBlueTableSchema mySchema;
IBlueTableSchema? mySchema;
private readonly string[] myOutputSchemaStringArray;
public CSVReaderStreamMapper(string outputSchema) {
@@ -162,6 +162,7 @@ namespace BCG {
}
public IBlueRow map(IBlueRow row) {
ArgumentNullException.ThrowIfNull(mySchema);
object[] vals = new object[row.schema.columnInfo.Length];
row.Values.CopyTo(vals, 0);
for (int i = 0; i < mySchema.columnInfo.Length; i++) {
@@ -1,5 +1,7 @@
using Bts.ColumnSortExpressions;
using mbts.utils;
using System;
using System.Collections;
using System.Collections.Generic;
@@ -39,7 +41,7 @@ public class SortRow : IComparable, IBlueRow {
var otherVal = otherRow[col.Name];
int rv = myIsNull(this[col.Name]) ?
(myIsNull(otherVal) ? 0 : -1) :
(myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal));
(myIsNull(otherVal) ? 1 : ExceptionAssert.of_type<IComparable>(this[col.Name]).CompareTo(otherVal));
rv *= col.IsAscending ? 1 : -1;
return rv;
}
@@ -1,8 +1,7 @@
using System;
using System.Diagnostics;
using System.IO;
using BCG.Utilities.General;
using BCG.Utilities.General;
using System;
using System.IO;
namespace BCG
{
@@ -49,8 +48,7 @@ namespace BCG
#region IWorker
public void doWork()
{
if ((myNextWrite <= DateTime.Now && myGotUpdate))
{
if (myNextWrite <= DateTime.Now && myGotUpdate) {
myGotUpdate = false;
myNextWrite = DateTime.Now.AddSeconds(myWriteFreqSecs);
myWrite(myStream.rows);
@@ -96,7 +94,7 @@ namespace BCG
rowStr += DELIM;
}
rowStr += row[columnName].ToString();
rowStr += row[columnName]?.ToString() ?? string.Empty;
}
writer.WriteLine(rowStr);
@@ -139,7 +137,7 @@ namespace BCG
private int myWriteFreqSecs;
private DateTime myNextWrite;
private bool myGotUpdate;
private string mySchemaStr;
private string? mySchemaStr;
#endregion
}
}
@@ -1,9 +1,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Bts.DataStreamCore;
using System;
using System.Collections;
using System.Collections.Generic;
namespace BCG
{
@@ -11,7 +10,7 @@ namespace BCG
{
private ViewStream _src;
private string _time_stamp_field;
private IBlueTableSchema _schema;
private IBlueTableSchema? _schema;
private string _latency_field;
public LatencyStream(ViewStream input, string time_stamp_field, string latency_field)
@@ -38,10 +37,10 @@ namespace BCG
public IBlueRow map(IBlueRow row)
{
var ht = new Hashtable(row);
if (row[_time_stamp_field].Equals(DBNull.Value))
if (Equals(row[_time_stamp_field], DBNull.Value))
ht[_latency_field] = DBNull.Value;
else
ht[_latency_field] = (DateTime.Now - (DateTime)row[_time_stamp_field]).TotalMilliseconds;
ht[_latency_field] = (DateTime.Now - row.GetValue<DateTime>(_time_stamp_field)).TotalMilliseconds;
return new BlueRow(_schema, ht);
}
@@ -1,7 +1,8 @@
using System;
using Bts.DataStreamCore;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BCG
{
@@ -42,8 +43,8 @@ namespace BCG
BlueTable result = new BlueTable(dp.schema);
foreach (IBlueRow r in dp.newRows)
{
string key = r[myCol_Key].ToString();
DateTime dt = (DateTime) r[myCol_Time];
var key = r[myCol_Key]?.ToString() ?? string.Empty;
var dt = r.GetValue<DateTime>(myCol_Time);
double value = Convert.ToDouble(r[myCol_Value]);
object[] rowValues = new object[r.Count];
@@ -68,9 +69,7 @@ namespace BCG
private double myUpdateValue(string key, DateTime time, double value)
{
SortedList<DateTime, double> seriesValues;
if (!myKeyToTimeToValue.TryGetValue(key, out seriesValues))
{
if (!myKeyToTimeToValue.TryGetValue(key, out SortedList<DateTime, double>? seriesValues)) {
seriesValues = new SortedList<DateTime, double>();
myKeyToTimeToValue[key] = seriesValues;
}
@@ -1,3 +1,7 @@
using Bts.DataStreamCore;
using mbts.utils;
using System;
using System.Collections;
@@ -12,7 +16,7 @@ namespace BCG
private int myId = 0;
private IDictionary myLeftIndex = new Hashtable (),
myRightIndex = new Hashtable ();
private IBlueTableSchema mySchema;
private IBlueTableSchema? mySchema;
public AppendCollection () {}
@@ -24,15 +28,15 @@ namespace BCG
IDictionary index = isLeft ? myLeftIndex : myRightIndex;
foreach (IBlueRow r in dp.deletedRows)
{
IBlueRow dr = index[r["id"]] as IBlueRow;
var dr = ExceptionAssert.of_type<IBlueRow>(index[r.Id()]);
dRows.addRow (dr);
index.Remove (r["id"]);
index.Remove (r.Id());
}
foreach (IBlueRow r in dp.addedRows)
{
IBlueRow ar = new AppendRow (r, myId++, mySchema);
aRows.addRow (ar);
index.Add (r["id"], ar);
index.Add(r.Id(), ar);
}
return new DataPacket (this, aRows, dRows);
}
@@ -43,7 +47,8 @@ namespace BCG
public bool containsID (int id) {throw new ApplicationException ("IBlueRow is readonly");}
public IBlueTableSchema schema {get {return mySchema;}}
public IBlueTableSchema schema =>
ExceptionAssert.not_null(mySchema);
#endregion
@@ -60,15 +65,14 @@ namespace BCG
array.SetValue (r, i++);
}
public object SyncRoot {get {return null;}}
public object SyncRoot => this;
#endregion
#region IEnumerable Members
public System.Collections.IEnumerator GetEnumerator()
{
return new MyEnumerator (myLeftIndex, myRightIndex);
public System.Collections.IEnumerator GetEnumerator() {
return new MyEnumerator(myLeftIndex, myRightIndex);
}
private class MyEnumerator : IEnumerator
@@ -1,11 +1,12 @@
using System;
using mbts.utils;
namespace BCG
{
public class AppendStreamModified : ViewStream
{
private AppendCollection myColl = new AppendCollection ();
private ViewStream myLeftStream, myRightstream;
ViewStream? myLeftStream;
ViewStream? myRightstream;
public AppendStreamModified (ViewStream leftStream, ViewStream rightStream)
{
@@ -19,7 +20,7 @@ namespace BCG
mySetRealSource (vss[0]);
else
{
AppendStreamModified rv = null;
AppendStreamModified? rv = null;
foreach (ViewStream s in vss)
rv = rv == null ? new AppendStreamModified (new ViewStream[] {s}) : new AppendStreamModified (rv, s);
mySetRealSource (rv);
@@ -28,24 +29,26 @@ namespace BCG
protected override void myOnFirstSubscriber()
{
myLeftStream.onChange += myHandleLeftChanges;
myRightstream.onChange += myHandleRightChanges;
ExceptionAssert.not_null(myLeftStream).onChange += myHandleLeftChanges;
ExceptionAssert.not_null(myRightstream).onChange += myHandleRightChanges;
}
protected override void myOnLastSubscriber()
{
myLeftStream.onChange -= myHandleLeftChanges;
myRightstream.onChange -= myHandleRightChanges;
ExceptionAssert.not_null(myLeftStream).onChange -= myHandleLeftChanges;
ExceptionAssert.not_null(myRightstream).onChange -= myHandleRightChanges;
}
private void myHandleLeftChanges (object newValue)
{
myFireEvent (myGetDataPacket (newValue as DataPacket, true));
var dp = ExceptionAssert.Argument.Is<DataPacket>(newValue);
myFireEvent(myGetDataPacket(dp, true));
}
private void myHandleRightChanges (object newValue)
{
myFireEvent (myGetDataPacket (newValue as DataPacket, false));
var dp = ExceptionAssert.Argument.Is<DataPacket>(newValue);
myFireEvent(myGetDataPacket(dp, false));
}
private DataPacket myGetDataPacket (DataPacket dp, bool isLeft)
@@ -1,3 +1,4 @@
#nullable enable
using System;
using System.Text.Json.Serialization;
@@ -6,10 +7,10 @@ namespace BCG {
/// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
/// </summary>
[Serializable]
public class ColumnInfo : IComparable {
public class ColumnInfo : IEquatable<ColumnInfo>, IComparable<ColumnInfo>, IComparable {
//the default column info does not cach values in a lookup table, therefore it is in group 0
public ColumnInfo(string colName, int colNo, Type colType, object colValue = null) {
public ColumnInfo(string colName, int colNo, Type colType, object? colValue = null) {
myColName = colName;
myColNo = colNo;
myColType = colType;
@@ -27,22 +28,23 @@ namespace BCG {
type == typeof(string) ||
type == typeof(DateTime);
public int CompareTo(object obj) {
int otherColNo = ((ColumnInfo)(obj)).colNo;
if (myColNo < otherColNo)
return -1;
else if (myColNo > otherColNo)
return 1;
return 0;
}
public int CompareTo(ColumnInfo? other) =>
other is null ? 1 : colNo.CompareTo(other.colNo);
public override bool Equals(object obj) {
if (obj is ColumnInfo) {
ColumnInfo other = obj as ColumnInfo;
return (colNo == other.colNo) && (colName == other.colName) && (colType == other.colType);
}
return false;
}
public int CompareTo(object? obj) =>
CompareTo(obj as ColumnInfo);
public bool Equals(ColumnInfo? other) =>
other is not null
&& colNo == other.colNo
&& colType == other.colType
&& colName == other.colName;
public override bool Equals(object? obj) =>
Equals(obj as ColumnInfo);
public override int GetHashCode() =>
HashCode.Combine(colNo, colType, colName);
public override string ToString() {
return colName + " " + colNo + " " + colType;
@@ -55,7 +57,7 @@ namespace BCG {
private int myColNo;
private string myColName;
private Type myColType;
private object myColValue;
private object? myColValue;
[JsonPropertyName("colNo")]
public int colNo { get { return myColNo; } }
@@ -67,6 +69,6 @@ namespace BCG {
public Type colType { get { return myColType; } }
[JsonIgnore]
public object colValue {get{return myColValue;}}
public object? colValue {get{return myColValue;}}
}
}
@@ -1,6 +1,7 @@
using System;
#nullable enable
using BCG;
using System.Collections;
using System;
using System.IO;
// common place for all serialization interfaces
@@ -14,7 +15,7 @@ namespace bts {
// other operations can be expressed with this
MULTIPLE_REMOVE_ADD = 1,
}
public class CreateMessage {
public class CreateMessage : IEquatable<CreateMessage> {
public IBlueRowCollection rows;
public CreateMessage(IBlueRowCollection rows_) {
rows = rows_;
@@ -30,15 +31,19 @@ namespace bts {
}
return rv;
}
public override bool Equals(object obj) {
if (obj != null && obj is CreateMessage) {
var other = obj as CreateMessage;
return BlueDataUtility.compute_equality(rows, other.rows);
}
return false;
}
public bool Equals(CreateMessage? other) =>
other is not null && BlueDataUtility.compute_equality(rows, other.rows);
public override bool Equals(object? obj) =>
Equals(obj as CreateMessage);
public override int GetHashCode() =>
// Not the most elegant hash, but it works with the Equals fn. CreateMessages that are
// 'Equal' are guaranteed to have the same hash.
HashCode.Combine(rows.Count, rows.schema.columnInfo.Length);
}
public class MultipleRemoveAddMessage {
public class MultipleRemoveAddMessage : IEquatable<MultipleRemoveAddMessage> {
public IBlueRowCollection removed_rows;
public IBlueRowCollection added_rows;
public MultipleRemoveAddMessage(IBlueRowCollection removed_rows_, IBlueRowCollection added_rows_) {
@@ -56,18 +61,23 @@ namespace bts {
}
return rv;
}
public override bool Equals(object obj) {
if (obj != null && obj is MultipleRemoveAddMessage) {
MultipleRemoveAddMessage other = obj as MultipleRemoveAddMessage;
return BlueDataUtility.compute_equality(removed_rows, other.removed_rows) &&
BlueDataUtility.compute_equality(added_rows, other.added_rows);
}
return false;
}
public bool Equals(MultipleRemoveAddMessage? other) =>
other is not null
&& BlueDataUtility.compute_equality(removed_rows, other.removed_rows)
&& BlueDataUtility.compute_equality(added_rows, other.added_rows);
public override bool Equals(object? obj) =>
Equals(obj as MultipleRemoveAddMessage);
public override int GetHashCode() =>
// Not the most elegant hash, but it works with the Equals fn. CreateMessages that are
// 'Equal' are guaranteed to have the same hash.
HashCode.Combine(
removed_rows.Count, removed_rows.schema.columnNames.Length, added_rows.Count, added_rows.schema.columnInfo.Length);
}
public StreamOp op;
public readonly CreateMessage create_message = null;
public readonly MultipleRemoveAddMessage multiple_remove_add_message = null;
public readonly CreateMessage? create_message = null;
public readonly MultipleRemoveAddMessage? multiple_remove_add_message = null;
public StreamIOMessage(CreateMessage create_message_) {
op = StreamOp.CREATE;
@@ -83,29 +93,26 @@ namespace bts {
public override string ToString() {
switch (op) {
case StreamIOMessage.StreamOp.CREATE:
return create_message.ToString();
case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD:
return multiple_remove_add_message.ToString();
case StreamOp.CREATE:
return create_message!.ToString();
case StreamOp.MULTIPLE_REMOVE_ADD:
return multiple_remove_add_message!.ToString();
}
bts.Log.assert(false);
return String.Empty;
}
public override bool Equals(object obj) {
if (obj != null && obj is StreamIOMessage) {
StreamIOMessage other = obj as StreamIOMessage;
if (op == other.op) {
switch (op) {
case StreamOp.CREATE:
return create_message.Equals(other.create_message);
case StreamOp.MULTIPLE_REMOVE_ADD:
return multiple_remove_add_message.Equals(other.multiple_remove_add_message);
}
}
}
return false;
}
public bool Equals(StreamIOMessage? other) =>
op == other?.op && op switch {
StreamOp.CREATE => create_message!.Equals(other.create_message),
StreamOp.MULTIPLE_REMOVE_ADD => multiple_remove_add_message!.Equals(other.multiple_remove_add_message),
_ => false,
};
public override bool Equals(object? obj) =>
Equals(obj as StreamIOMessage);
public override int GetHashCode() =>
HashCode.Combine((int)op, create_message?.GetHashCode() ?? multiple_remove_add_message?.GetHashCode() ?? 0);
}
#region serialization
// This helps putting stream callbacks (op + data) on the wire, to send it to the pubsubserver.
@@ -134,4 +141,4 @@ namespace bts {
}
#endregion
}
}
@@ -11,45 +11,45 @@ namespace bts.BlueOrb {
public static class ContractUtils {
public class FutureTradableContract {
public string symbol;
public string? symbol;
public DateTime expiration;
public Exchange exchange;
public string contract_text;
public string? contract_text;
}
public class FutureSpreadContract {
public List<Tuple<double, FutureTradableContract>> legs;
public string symbol {
public List<Tuple<double, FutureTradableContract>>? legs;
public string? symbol {
get {
return legs == null || legs.Count == 0 ? string.Empty : legs.ElementAt(0).Item2.symbol;
}
}
public string contract_text;
public string? contract_text;
}
public class FutureContinousContract {
public string symbol;
public string? symbol;
public Exchange exchange;
}
// this is for all future conytracts belonging to a symbol
public class FutureWildcardContract {
public string symbol;
public string? symbol;
public Exchange exchange;
}
public class FutureContract {
public FutureWildcardContract wildcard_contract = null;
public FutureTradableContract future_tradable_contract = null;
public FutureSpreadContract spread_tradable_contract = null;
public FutureContinousContract continous_constract = null;
public FutureWildcardContract? wildcard_contract = null;
public FutureTradableContract? future_tradable_contract = null;
public FutureSpreadContract? spread_tradable_contract = null;
public FutureContinousContract? continous_constract = null;
public FutureContract(string txt) {
wildcard_contract = parse_wildcard_contract(txt);
if (wildcard_contract != null)
return;
FutureTradableContract fc = parse_future_contract(txt);
var fc = parse_future_contract(txt);
if (fc != null) {
future_tradable_contract = fc;
return;
}
FutureSpreadContract sc = parse_future_spread_contract(txt);
var sc = parse_future_spread_contract(txt);
if (sc != null) {
spread_tradable_contract = sc;
return;
@@ -63,7 +63,7 @@ namespace bts.BlueOrb {
public bool is_future_tradable_contract => future_tradable_contract != null;
public bool is_spread_tradable_contract => spread_tradable_contract != null;
public string symbol() {
public string? symbol() {
if (wildcard_contract != null)
return wildcard_contract.symbol;
if (continous_constract != null) {
@@ -76,7 +76,7 @@ namespace bts.BlueOrb {
Log.assert(false);
return string.Empty;
}
public string contract_text {
public string? contract_text {
get {
if (future_tradable_contract != null)
return future_tradable_contract.contract_text;
@@ -87,7 +87,7 @@ namespace bts.BlueOrb {
}
}
#region Parsing
public static FutureWildcardContract parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) {
public static FutureWildcardContract? parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) {
if (contract_txt.EndsWith("*")) {
contract_txt = contract_txt.Replace("*", string.Empty);
return new FutureWildcardContract() {
@@ -97,7 +97,7 @@ namespace bts.BlueOrb {
}
return null;
}
public static FutureTradableContract parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) {
public static FutureTradableContract? parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) {
// Validate the future string length
if (!contract_txt.All(c => char.IsLetterOrDigit(c)))
return null;
@@ -153,7 +153,7 @@ namespace bts.BlueOrb {
return futureContract;
}
// Method to parse a futures spread, e.g. "CLX4-CLZ4"
public static FutureSpreadContract parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) {
public static FutureSpreadContract? parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) {
// Split the string by '-' to handle each leg
var spreadParts = contract_txt.Split('-');
if (spreadParts.Length < 2) {
@@ -201,11 +201,11 @@ namespace bts.BlueOrb {
#endregion
public static ContractType get_type(string contract_txt) {
FutureTradableContract fc = parse_future_contract(contract_txt);
var fc = parse_future_contract(contract_txt);
if (fc != null) {
return ContractType.Future;
}
FutureSpreadContract sc = parse_future_spread_contract(contract_txt);
var sc = parse_future_spread_contract(contract_txt);
if (sc != null) {
return ContractType.FutureSpread;
}
@@ -220,13 +220,13 @@ namespace bts.BlueOrb {
}
public static Tuple<ContractCategory, string> get_symbol_category(string symbol) {
FutureTradableContract fc = parse_future_contract(symbol);
var fc = parse_future_contract(symbol);
if (fc != null) {
return Tuple.Create(ContractCategory.FutureString, fc.symbol);
return Tuple.Create(ContractCategory.FutureString, fc.symbol ?? string.Empty);
}
var sc = parse_future_spread_contract(symbol);
if (sc != null) {
return Tuple.Create(ContractCategory.FutureString, sc.symbol);
return Tuple.Create(ContractCategory.FutureString, sc.symbol ?? string.Empty);
}
return Tuple.Create(ContractCategory.Symbol, symbol);
}
@@ -1,8 +1,6 @@
namespace bts.BlueOrb {
using bts.SilverPine;
using BCG;
public class DateTimeRange {
public readonly DateTime range_beg;
public readonly DateTime range_end;
@@ -18,7 +16,7 @@ namespace bts.BlueOrb {
}
public class FutureBarsLoader {
public static string get_db_table_name(BarSize bar_size, string symbol, bool is_continous) {
public static string? get_db_table_name(BarSize bar_size, string symbol, bool is_continous) {
switch (bar_size) {
case BarSize.ONE_DAY: {
if (is_continous) {
@@ -68,22 +66,22 @@ namespace bts.BlueOrb {
string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
return new SQLiteStream(SQLiteHelper.db_name, sql_str);
};
string symbol = future_contract.symbol();
var symbol = future_contract.symbol() ?? string.Empty;
if (future_contract.wildcard_contract != null) {
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, false) ?? string.Empty;
ViewStream rs = get_sqlite_str(table_name);
//rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
mySetRealSource(rs);
return;
}
if (future_contract.is_continous_contract) {
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true);
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true) ?? string.Empty;
ViewStream rs = get_sqlite_str(table_name);
//rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
mySetRealSource(rs);
return;
} else {
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, false) ?? string.Empty;
ViewStream rs = get_sqlite_str(table_name);
//rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
//TODO: filter it
@@ -115,20 +113,20 @@ namespace bts.BlueOrb {
string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
return new SQLiteStream(SQLiteHelper.db_name, sql_str);
};
string symbol = future_contract.symbol();
var symbol = future_contract.symbol() ?? string.Empty;
if (future_contract.wildcard_contract != null) {
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, false) ?? string.Empty;
ViewStream rs = get_sqlite_str(table_name);
mySetRealSource(rs);
return;
}
if (future_contract.is_continous_contract) {
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true);
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true) ?? string.Empty;
ViewStream rs = get_sqlite_str(table_name);
mySetRealSource(rs);
return;
} else {
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, false) ?? string.Empty;
ViewStream rs = get_sqlite_str(table_name);
// TODO: filter
rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'");
@@ -1,6 +1,8 @@
namespace bts {
using BCG;
using mbts.utils;
using System;
using System.Collections.Generic;
using System.Net;
@@ -8,7 +10,7 @@ namespace bts {
using System.Text.Json.Nodes;
[DataStreamUserRespresentation("benzingaNews")]
public class BenzingaNewsStream : ViewStream {
private IBlueTableSchema _schema = null;
private IBlueTableSchema? _schema;
private int _id = 0;
@@ -63,14 +65,14 @@ namespace bts {
}
foreach (JsonNode jnode in json_objs) {
if (jnode is JsonObject json_object) {
string symbol = jnode["symbol"] == null ? String.Empty : jnode["symbol"].ToString();
string exchange = jnode["exchange"] == null ? String.Empty : jnode["exchange"].ToString();
string symbol = jnode["symbol"]?.ToString() ?? string.Empty;
string exchange = jnode["exchange"]?.ToString() ?? string.Empty;
MutableBlueRow nr = new MutableBlueRow(_schema, r);
nr.setValue("id", _id++);
nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
nr.setValue("symbol", symbol);
nr.setValue("exchange", exchange);
nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
nr.setValue("is_primary_exchange", bool.Parse(ExceptionAssert.not_null(jnode["primary"]).ToString()));
expanded_rows.Add(nr);
}
}
@@ -84,4 +86,4 @@ namespace bts {
mySetRealSource(output_stream);
}
}
}
}
@@ -2,8 +2,6 @@ namespace bts {
using BCG;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
//[DataStreamUserRespresentation("benzingaNewsSymbols")]
public class BenzingaNewsSymbolStateStream : ViewStream, INonIncGroupByEvaluator {
@@ -11,23 +9,23 @@ namespace bts {
ViewStream r = new NonIncGroupByStream(news_str, this);
mySetRealSource(r);
}
public BenzingaNewsSymbolStateStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl) :
this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) {
}
public BenzingaNewsSymbolStateStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl)
: this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) { }
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
BlueTable rv = null;
BlueTable? rv = null;
foreach (var kv in grouped_rows) {
var symbol_rows = kv.Value;
IBlueRow state_row = BlueDataUtility.find_max(symbol_rows, "created_at");
rv = rv == null ? new BlueTable(state_row.schema) : rv;
rv ??= new(state_row.schema);
rv.addRow(state_row);
}
return rv;
return rv ?? BlueTable.Empty("BenzingaNewsSymbolState");
}
public string[] get_group_by_cols() {
return new string[] { "symbol" };
}
}
}
}
@@ -3,6 +3,7 @@ using bts.config;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
namespace BCG {
@@ -21,7 +22,9 @@ namespace BCG {
}
public class RenkoCalculator {
public readonly double brick_size;
public RenkoBrick current_brick = null;
public RenkoBrick? current_brick = null;
[MemberNotNull(nameof(current_brick))]
private bool _add_price(double price) {
if (current_brick == null) {
current_brick = new RenkoBrick {
@@ -309,4 +312,4 @@ namespace BCG {
return wma_values;
}
}
}
}
@@ -3,6 +3,8 @@
using bts.md;
using mbts.utils;
using System;
using System.Collections.Generic;
@@ -13,7 +15,7 @@
public class SpatialArbStatsStream : ViewStream{
private ViewStream _spatial_pairs_stream;
private ViewStreamDatabase _vsd;
private ViewStream _tot_str = null;
private ViewStream? _tot_str = null;
private HashSet<string> _key_map = new HashSet<string>();
private ViewStream _bbo_stream;
private readonly TimeSpan _window_sz, _sampling_period_sz;
@@ -89,7 +91,7 @@
//_tot_str = new SortStream(_tot_str, "-n_trade_ticks");
_tot_str.onChange += _handle_tot_stream;
DataPacket dp = dp_obj as DataPacket;
var dp = ExceptionAssert.Argument.Is<DataPacket>(dp_obj);
foreach (IBlueRow r in dp.newRows) {
string s1 = r.get<string>("s1");
string s2 = r.get<string>("s2");
@@ -4,6 +4,8 @@
using bts.api;
using bts.md;
using mbts.utils;
using System;
using System.Collections.Generic;
@@ -67,7 +69,7 @@
private void _main_stream_onchange(object dp_obj) {
_key_map.Clear();
DataPacket dp = dp_obj as DataPacket;
var dp = ExceptionAssert.Argument.Is<DataPacket>(dp_obj);
BlueTable table = new BlueTable(_schema);
var rows = dp.newRows.to_list();
foreach (var r1 in rows) {
@@ -1,30 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using MathNet.Numerics.Statistics;
using MathNet.Numerics.LinearRegression;
using System.Collections.Generic;
using System.Linq;
using MathNet.Numerics.Statistics;
using MathNet.Numerics.LinearAlgebra;
using Amazon.Runtime;
using System;
using System.Collections.Generic;
using System.Linq;
using MathNet.Numerics.Statistics;
using System;
using System.Collections.Generic;
using System.Linq;
using MathNet.Numerics.Statistics;
using System;
using System.Collections.Generic;
using System.Linq;
using MathNet.Numerics.Statistics;
// stats calculator
using System;
using System.Collections.Generic;
using System.Linq;
using MathNet.Numerics.Statistics;
/// <summary>
/// Provides evaluation metrics to assess the quality of a synthetic spread for mean-reversion trading.
@@ -35,13 +12,13 @@ public static class SpreadPerformanceEvaluator
public static class SpreadStatistics {
public class SpreadStatsResult {
public Dictionary<string, double> bid_zscore_distribution { get; set; }
public Dictionary<string, double> ask_zscore_distribution { get; set; }
public Dictionary<string, double> mid_zscore_distribution { get; set; }
public Dictionary<string, double> bid_zscore_distribution { get; set; } = new(0);
public Dictionary<string, double> ask_zscore_distribution { get; set; } = new(0);
public Dictionary<string, double> mid_zscore_distribution { get; set; } = new(0);
public DescriptiveStatistics bid_stats { get; set; }
public DescriptiveStatistics ask_stats { get; set; }
public DescriptiveStatistics mid_stats { get; set; }
public DescriptiveStatistics bid_stats { get; set; } = new();
public DescriptiveStatistics ask_stats { get; set; } = new();
public DescriptiveStatistics mid_stats { get; set; } = new();
}
private static readonly string[] _zscore_labels = new[]
@@ -1,10 +1,12 @@
namespace bts.BlueOrb {
using BCG;
using mbts.utils;
using System.Diagnostics;
public class SpreadPerformanceStream : ViewStream {
ViewStream _spread_price_stream = null;
ViewStream _spread_price_stream;
private readonly IBlueTableSchema _schema;
private readonly int _window_secs;
private readonly double _buy_threshold, _sell_threshold, _flat_threshold;
@@ -70,7 +72,7 @@ namespace bts.BlueOrb {
}
private void _handle_spread_prices(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
var dp = ExceptionAssert.Argument.Is<DataPacket>(dp_obj);
List<Tuple<DateTime, double, double, double, double>> spread_ts_prices = new List<Tuple<DateTime, double, double, double, double>>();
DateTime min_time = DateTime.MaxValue; DateTime max_time = DateTime.MinValue;
var sw = Stopwatch.StartNew();
@@ -5,6 +5,8 @@ namespace bts.BlueOrb{
using bts;
using bts.md;
using mbts.utils;
using Spark.Core.Disposables;
using Spark.Core.Reactive;
@@ -103,11 +105,11 @@ namespace bts.BlueOrb{
new_state.addRow(r);
}
}
DataPacket dp = null;
DataPacket? dp;
if (myLastValue == null) {
dp = new DataPacket(new_state, new_state, new BlueTable(_bbo_schema));
} else {
DataPacket old_dp = myLastValue as DataPacket;
var old_dp = ExceptionAssert.of_type<DataPacket>(myLastValue);
IBlueRowCollection added_rows, del_rows;
BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
dp = new DataPacket(new_state, added_rows, del_rows);
@@ -26,7 +26,7 @@ namespace bts.BlueOrb{
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
Func<string, bool> is_future = (c) => ContractUtils.get_category(c) == ContractUtils.ContractCategory.FutureString;
BlueTable rv = null;
BlueTable? rv = null;
Func<List<IBlueRow>, IBlueRow> reduce = (rows) => {
var future_rows = rows.Where(r => is_future(r.get<string>("contract")));
IBlueRow rv = future_rows.ElementAt(0);
@@ -4,8 +4,12 @@ namespace bts.BlueOrb {
using bts;
using bts.api;
using bts.md;
using BCG;
using mbts.utils;
using Spark.Core;
using Spark.Core.Disposables;
using Spark.Core.Reactive;
@@ -13,18 +17,14 @@ namespace bts.BlueOrb {
using System.Collections.Generic;
using System.Linq;
using System.Reactive.Linq;
using SecurityKey = System.UInt32;
[DataStreamUserRespresentation("InstrumentsStream")]
public class InstrumentsStream : ViewStream, IWorker {
private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
private DateTime _last_poll_time = DateTime.MinValue;
private ViewStreamDatabase _vsd;
private int _id = 0;
IReadOnlyDictionary<uint, SecurityImmutable> _cached_state = null;
IReadOnlyDictionary<uint, SecurityImmutable>? _cached_state;
MarketDataClient _md_client;
private bool _dirty = false;
private readonly IBlueTableSchema _instrument_schema = BlueDataUtility.form_schema("instruments",
@@ -68,7 +68,7 @@ namespace bts.BlueOrb {
_dirty = true;
//_log_updates(state);
// just cacching in here, will populate the rows in _poll_scan and fire the datapacket
bts.Log.Gen.info($"InstrumentsStream._on_instruments state: {state.Count()}");
bts.Log.Gen.info($"InstrumentsStream._on_instruments state: {state.Count}");
_cached_state = state;
}
@@ -85,13 +85,13 @@ namespace bts.BlueOrb {
return;
BlueTable new_state = new BlueTable(_instrument_schema);
foreach (var kv in _cached_state) {
SecurityKey skey = kv.Key;
var skey = kv.Key;
int id = Convert.ToInt32(skey);
SecurityImmutable sec = kv.Value;
bts.api.SecurityType sec_type = sec.security_type();
object exp = DBNull.Value;
if (sec_type == bts.api.SecurityType.Future || sec_type == bts.api.SecurityType.Option)
exp = sec.security_expiration();
exp = sec.security_expiration().assert_value();
object strike = DBNull.Value;
if (sec_type == bts.api.SecurityType.Option)
strike = sec.strike();
@@ -101,11 +101,11 @@ namespace bts.BlueOrb {
IBlueRow r = new BlueRow(_instrument_schema, id, skey, sec.security_desc(), sec.security_type().ToString(), sec.display_exchange().ToString(), sec.Symbol, sec.Currency, sec.ContractSpec.CashPerPoint, exp, strike, cp);
new_state.addRow(r);
}
DataPacket dp = null;
DataPacket? dp;
if (myLastValue == null) {
dp = new DataPacket(new_state, new_state, new BlueTable(_instrument_schema));
} else {
DataPacket old_dp = myLastValue as DataPacket;
var old_dp = ExceptionAssert.of_type<DataPacket>(myLastValue);
IBlueRowCollection added_rows, del_rows;
BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
dp = new DataPacket(new_state, added_rows, del_rows);
@@ -167,8 +167,8 @@ namespace bts.BlueOrb {
return String.Empty;
}
private Source _sourse = Source.SQL; // get from config
private readonly string _location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:\\BlueOrb_Data\\":Path.Combine(Environment.GetEnvironmentVariable("HOME") , "BlueOrb_Data");
// private Source _sourse = Source.SQL; // get from config
// private readonly string _location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:\\BlueOrb_Data\\":Path.Combine(Environment.GetEnvironmentVariable("HOME") , "BlueOrb_Data");
private readonly string _db_name = "stream_data.db";
// the symbol is used ot retreive the correct table, the data flows as-is from the table
// with some name / type changes
@@ -1,44 +1,28 @@
namespace BCG {
using BCG.Utilities.General;
using bts.api;
using bts.api.domain;
using bts.api.synthetic;
using bts;
using bts.api.core;
using bts.md;
using bts.spark;
using bts.utils;
using mbts;
using mbts.co2;
using mbts.config;
using mbts.spark;
using mbts.ui;
using mbts.utils;
using Spark.Core.Data;
using Spark.Core.Disposables;
using Spark.Core.Reactive;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
using System.Reactive.Linq;
using bts.api.core;
using Spark.Api.MarketData;
using SecurityKey = System.UInt32;
using System.Data;
using System.Reflection.Metadata;
using bts;
[DataStreamUserRespresentation("price_stats")]
public class PriceStatsStream : ViewStream, IWorker {
private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
private DateTime _last_poll_time = DateTime.MinValue;
private ViewStreamDatabase _vsd;
private int _id = 0;
IReadOnlyDictionary<SecurityKey, PriceStats> _cached_state = null;
IReadOnlyDictionary<uint, PriceStats>? _cached_state = null;
MarketDataClient _md_client;
private bool _dirty = false;
private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo",
@@ -52,14 +36,13 @@ namespace BCG {
IDisposable? subscription;
// TODO: add cons for symbol, exchange etc
public PriceStatsStream()
: this(ViewStreamDatabase.instance) {
}
public PriceStatsStream(MarketDataClient mdClient)
: this(mdClient, ViewStreamDatabase.instance) { }
public PriceStatsStream(ViewStreamDatabase vsd) {
public PriceStatsStream(MarketDataClient mdClient, ViewStreamDatabase vsd) {
bts.Log.Gen.info("PriceStatsStream.cons");
_md_client = mdClient;
_vsd = vsd;
_md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
}
protected override void myOnFirstSubscriber() {
@@ -69,7 +52,7 @@ namespace BCG {
}
private void _log_updates(TableUpdate<PriceStats, uint> update) {
int i = 0;
// int i = 0;
// foreach (BookTuple bt in update.added) {
// bts.Log.Gen.info($"PriceStatsStream added: idx: {i++} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
// }
@@ -99,18 +82,18 @@ namespace BCG {
return;
BlueTable new_state = new BlueTable(_pstats_scheam);
foreach (var kv in _cached_state) {
SecurityKey skey = kv.Key;
var skey = kv.Key;
int id = Convert.ToInt32(skey);
PriceStats pstats = kv.Value;
var today = DateTime.Now.Date;
IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), today, pstats.open, pstats.close, pstats.settlement);
new_state.addRow(r);
}
DataPacket dp = null;
DataPacket? dp;
if (myLastValue == null) {
dp = new DataPacket(new_state, new_state, new BlueTable(_pstats_scheam));
} else {
DataPacket old_dp = myLastValue as DataPacket;
var old_dp = ExceptionAssert.of_type<DataPacket>(myLastValue);
IBlueRowCollection added_rows, del_rows;
BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
dp = new DataPacket(new_state, added_rows, del_rows);
@@ -85,7 +85,8 @@ namespace bts.BlueOrb {
.GetProperty("choices")[0]
.GetProperty("message")
.GetProperty("content")
.GetString();
.GetString()
?? string.Empty;
//return gpt_response;
DateTime t1 = DateTime.Now;
@@ -1,5 +1,6 @@
using BCG.Utilities.General;
using BCG;
using mbts.utils;
namespace bts {
/// <summary>
@@ -10,7 +11,7 @@ namespace bts {
public class PubSubServerStateStream : ViewStream, IWorker {
private IDictionary<string, TagPubSubInfo> _tag_map = new Dictionary<string, TagPubSubInfo>();
private readonly string _tag = "System_AllSubscriptions";
private IBlueTableSchema _schema = null;
private IBlueTableSchema? _schema;
private SubscriberCore _sub_core;
private ViewStreamDatabase _vsd;
@@ -47,12 +48,11 @@ namespace bts {
var updates = _sub_core.get_updates(TimeSpan.Zero);
//this is where the pubsub updates are converted to rows
if (updates.Count() > 0) {
int id = 0;
foreach (var u in updates) {
if (u.tag != _tag) {
continue;
}
var tpsi_list = TagPubSubInfo.deserialize(u.data);
var tpsi_list = TagPubSubInfo.deserialize(Log.notnull(u.data));
foreach (var tsi in tpsi_list) {
_tag_map[tsi.tag] = tsi;
}
@@ -7,8 +7,8 @@ using BCG.Utilities.General;
public class PubsubHeartbeatStream : ViewStream, IWorker {
private ViewStreamDatabase _vsd;
private SubscriberCore _subcore = new SubscriberCore();
private MutableBlueRow _row = null;
private readonly IBlueTableSchema _schema = null;
private MutableBlueRow? _row;
private readonly IBlueTableSchema? _schema;
public PubsubHeartbeatStream(ViewStreamDatabase vsd) {
_vsd = vsd;
_schema = BlueDataUtility.form_schema("PS_HBB",
@@ -5,12 +5,14 @@ namespace bts
using Bts.DataStreamCore.StreamFactories;
using mbts.utils;
public class TableMetaDataStream : ViewStream, IWorker
{
private class TableState
{
public ViewStream table_stream;
public IBlueTableSchema schema = null;
public IBlueTableSchema? schema;
public bool processed = false;
public TableState(ViewStream table_stream_)
{
@@ -19,8 +21,7 @@ namespace bts
}
private void _handle_table_update(object dp_obj)
{
DataPacket dp = dp_obj as DataPacket;
schema = dp.schema;
schema = ExceptionAssert.Argument.Is<DataPacket>(dp_obj).schema;
}
public void unsub()
{
@@ -93,7 +94,7 @@ namespace bts
private void _handle_base_stream(object dp_obj)
{
DataPacket dp = dp_obj as DataPacket;
var dp = ExceptionAssert.Argument.Is<DataPacket>(dp_obj);
foreach (IBlueRow r in dp.newRows)
{
string tag = r.get<string>("tag");
@@ -108,12 +109,7 @@ namespace bts
{
Log.assert(tag_name.StartsWith("DS_"));
string table_name = tag_name.Remove(0, 3);
TableState ts = null;
if (!_table_map.TryGetValue(table_name, out ts))
{
ts = new TableState(tableStreamFactory.Create(table_name));
_table_map.Add(table_name, ts);
}
_table_map.GetOrAdd(table_name, _ => new(tableStreamFactory.Create(table_name)));
}
}
}