From 288b7712db80e9f17f15a61c7cb39943dea0835d Mon Sep 17 00:00:00 2001 From: Taha Date: Wed, 9 Oct 2024 13:59:01 -0400 Subject: [PATCH] Added and modified more streams to address teh CME NL queries --- .vscode/launch.json | 7 +- .../DataStream/DataStream.csproj | 1 + .../Primitives/GroupBy/AverageGroupsStream.cs | 162 ++++++++-------- .../GroupBy/AverageNonIncGroupsStream.cs | 31 +++ .../Primitives/GroupBy/MaxStream.cs | 2 +- .../NonIncremental/NonIncGroupByStream.cs | 56 ++++-- .../NonIncremental/NonIncJoinBookkeeper.cs | 53 ++--- .../Join/NonIncremental/NonIncJoinSpec.cs | 57 +++--- .../Join/NonIncremental/NonIncJoinStream.cs | 82 +++----- .../Primitives/Literal/LiteralStream.cs | 34 +++- .../Primitives/Map/GlobalMapStream.cs | 72 +++++++ .../Primitives/Misc/TimeSeriesStream.cs | 1 - .../Primitives/SQL/SQLite/SQLiteHelper.cs | 65 +++++++ .../StaticCollectionStream/BlueTableStream.cs | 57 +++--- .../Time/DateTimeExpanstionStream.cs | 34 ++++ .../Timeseries/ColumnDifferenceStream.cs | 111 +++++++++++ .../Timeseries/ColumnEvolutionStream.cs | 51 +++++ .../BlueData/BlueDataUtility.cs | 100 ++++++++-- .../DataStreamCore/DataStreamCore.csproj | 1 + .../SharedStreams/Business/ContractUtils.cs | 181 ++++++++++++++++++ .../MarketData/AbsoluteVolumeStream.cs | 47 +++++ .../MarketData/Charting/ChartingHelper.cs | 114 +++++++++++ .../MarketData/Charting/DailyChangeStream.cs | 31 +++ .../Business/MarketData/Charting/EMAStream.cs | 54 ------ .../Business/MarketData/Charting/RSIStream.cs | 26 +++ .../Charting/RealizedVolatilityStream.cs | 0 .../MarketData/Charting/RenkoStream.cs | 87 --------- .../Business/MarketData/Charting/SMAStream.cs | 70 ++----- .../Charting/SecurityRenkoStream.cs | 49 ----- .../MarketData/Charting/TradeBarsStream.cs | 161 ---------------- .../MarketData/Charting/VWAPStream.cs | 56 ++++++ .../MarketData/FuturesActiveContractStream.cs | 78 ++++++++ .../Business/MarketData/FuturesRollStream.cs | 16 ++ .../MarketData/OHLCVPriceDiffStream.cs | 36 ++++ .../Business/MarketData/OHLCVStream.cs | 117 +++++++++++ .../MarketData/SettlementPriceStream.cs | 48 +++++ .../BarsAggregatingStream.cs | 32 ++++ .../TimSeriesOperations/BarsSearcherStream.cs | 32 ++++ .../TimeSeriesGroupByStream.cs | 130 ------------- src/lib/config/profiles/default/toggles.conf | 7 +- 40 files changed, 1564 insertions(+), 785 deletions(-) create mode 100644 src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageNonIncGroupsStream.cs create mode 100644 src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs create mode 100644 src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs create mode 100644 src/DataStreamSuite/DataStream/Primitives/Time/DateTimeExpanstionStream.cs create mode 100644 src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs create mode 100644 src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnEvolutionStream.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/AbsoluteVolumeStream.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/ChartingHelper.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/DailyChangeStream.cs delete mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/EMAStream.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RSIStream.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RealizedVolatilityStream.cs delete mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RenkoStream.cs delete mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SecurityRenkoStream.cs delete mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/TradeBarsStream.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesRollStream.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVPriceDiffStream.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/BarsAggregatingStream.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/BarsSearcherStream.cs delete mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs diff --git a/.vscode/launch.json b/.vscode/launch.json index 224b1ff56..800b9d764 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,6 +1,11 @@ { "version": "0.2.0", "configurations": [ + { + "name": ".NET Core Attach", + "type": "coreclr", + "request": "attach" + }, { "name": "DataStreamPublisher", "type": "coreclr", @@ -138,4 +143,4 @@ "default": "http://localhost:8000/config", }, ] -} +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/DataStream.csproj b/src/DataStreamSuite/DataStream/DataStream.csproj index d101ba8cc..8c8ed8d6b 100644 --- a/src/DataStreamSuite/DataStream/DataStream.csproj +++ b/src/DataStreamSuite/DataStream/DataStream.csproj @@ -20,6 +20,7 @@ + diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageGroupsStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageGroupsStream.cs index c9ccf45a5..8ce1a3159 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageGroupsStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageGroupsStream.cs @@ -1,102 +1,102 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; +// using System; +// using System.Collections.Generic; +// using System.Linq; +// using System.Text; -namespace BCG -{ +// namespace BCG +// { - public class AverageGroupsStream : ViewStream - { - public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn) - { +// public class AverageGroupsStream : ViewStream +// { +// public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn) +// { - myColToAverage = colToAverage; - myInput = input; - myKeyColumn = keyColumn; - } +// myColToAverage = colToAverage; +// myInput = input; +// myKeyColumn = keyColumn; +// } - private ViewStream myInput; - private string myColToAverage; - private BlueTableSchema mySchema; - private string myKeyColumn; +// private ViewStream myInput; +// private string myColToAverage; +// private BlueTableSchema mySchema; +// private string myKeyColumn; - protected override void myOnFirstSubscriber() - { - myInput.onChange += new ChangeDelegate(myInput_onChange); - } - protected override void myOnLastSubscriber() - { - myInput.onChange -= myInput_onChange; - } - private void myInput_onChange(object newValue) - { - if (mySchema == null) - myBuildSchema(((DataPacket)newValue).schema); +// protected override void myOnFirstSubscriber() +// { +// myInput.onChange += new ChangeDelegate(myInput_onChange); +// } +// protected override void myOnLastSubscriber() +// { +// myInput.onChange -= myInput_onChange; +// } +// private void myInput_onChange(object newValue) +// { +// if (mySchema == null) +// myBuildSchema(((DataPacket)newValue).schema); - IBlueRowCollection newRows = myEvaluate((DataPacket)newValue); +// IBlueRowCollection newRows = myEvaluate((DataPacket)newValue); - if (myLastValue == null) - myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema))); - else - { - IBlueRowCollection added, deleted; - BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added); - myFireEvent(new DataPacket(newRows, added, deleted)); - } - } +// if (myLastValue == null) +// myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema))); +// else +// { +// IBlueRowCollection added, deleted; +// BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added); +// myFireEvent(new DataPacket(newRows, added, deleted)); +// } +// } - private void myBuildSchema(IBlueTableSchema schema) - { - List columns = new List(); +// private void myBuildSchema(IBlueTableSchema schema) +// { +// List columns = new List(); - foreach (ColumnInfo ci in schema.columnInfo) - columns.Add(ci); +// foreach (ColumnInfo ci in schema.columnInfo) +// columns.Add(ci); - columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double))); - mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray()); - } +// columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double))); +// mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray()); +// } - private IBlueRowCollection myEvaluate(DataPacket packet) - { +// private IBlueRowCollection myEvaluate(DataPacket packet) +// { - IDictionary colToRunningTotal = new Dictionary(); - IDictionary colToCount = new Dictionary(); - IDictionary colToLastRow = new Dictionary(); +// IDictionary colToRunningTotal = new Dictionary(); +// IDictionary colToCount = new Dictionary(); +// IDictionary colToLastRow = new Dictionary(); - foreach (IBlueRow r in packet.newRows) - { - double thisVal = Convert.ToDouble(r[myColToAverage]); +// foreach (IBlueRow r in packet.newRows) +// { +// double thisVal = Convert.ToDouble(r[myColToAverage]); - string keyVal = r[myKeyColumn].ToString(); - if (!colToCount.ContainsKey(keyVal)) - { - colToRunningTotal[keyVal] = thisVal; - colToCount[keyVal] = 1; - } - else - { - colToRunningTotal[keyVal] += thisVal; - colToCount[keyVal] += 1; - } - colToLastRow[keyVal] = r; - } +// string keyVal = r[myKeyColumn].ToString(); +// if (!colToCount.ContainsKey(keyVal)) +// { +// colToRunningTotal[keyVal] = thisVal; +// colToCount[keyVal] = 1; +// } +// else +// { +// colToRunningTotal[keyVal] += thisVal; +// colToCount[keyVal] += 1; +// } +// colToLastRow[keyVal] = r; +// } - BlueTable result = new BlueTable(mySchema); - foreach (string key in colToCount.Keys) - { - object[] values = new object[mySchema.columnInfo.Length]; +// BlueTable result = new BlueTable(mySchema); +// foreach (string key in colToCount.Keys) +// { +// object[] values = new object[mySchema.columnInfo.Length]; - colToLastRow[key].CopyTo(values, 0); - values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0; - result.addRow(new BlueRow(mySchema, values)); - } +// colToLastRow[key].CopyTo(values, 0); +// values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0; +// result.addRow(new BlueRow(mySchema, values)); +// } - return result; - } +// return result; +// } - } -} +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageNonIncGroupsStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageNonIncGroupsStream.cs new file mode 100644 index 000000000..a5daa62ce --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageNonIncGroupsStream.cs @@ -0,0 +1,31 @@ +namespace bts.SilverPine { + using BCG; + + using System; + using System.Collections.Generic; + using System.Linq; + + [DataStreamUserRespresentation("Average")] + public class AverageNonIncGroupsStream : ViewStream { + public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) { + NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols, + (g_rows) => _find_mean(inp_col_name, g_rows)); + mySetRealSource(rs); + } + + private List _find_mean(string inp_col_name, List g_rows) { + IBlueRow inp_row = g_rows.ElementAt(0); + IEnumerable inp_vals = BlueDataUtility.get_colvalues(g_rows, inp_col_name); + double sum = 0.0; + inp_vals.ToList().ForEach(d => sum += d); + string out_col_name = $"average_{inp_col_name}"; + double average = sum / inp_vals.Count(); + IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name); + MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row); + out_row.setValue(out_col_name, average); + List rv = new List(); + rv.Add(out_row); + return rv; + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs index 8f55cab92..f87da6ebc 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs @@ -10,6 +10,6 @@ namespace BCG { } public MaxStream(ViewStream stream, string outputCol) - : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { } + : this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { } } } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs index 2e0b28992..c8b60b508 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs @@ -1,13 +1,37 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Linq; using bts; -namespace BCG -{ - public class NonIncGroupByStream : ViewStream - { +namespace BCG { + public class NonIncGroupByStream : ViewStream { + private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator { + Func _groupby_cols_func; + Func, List> _evaluate_func; + public DelNonIncGroupByEvaluator(Func groupby_cols_func, + Func, List> evaluate_func) { + _groupby_cols_func = groupby_cols_func; + _evaluate_func = evaluate_func; + } + public IBlueRowCollection evaluate(IDictionary> grouped_rows) { + BlueTable rv = null; + foreach (var kv in grouped_rows) { + List g_rows = kv.Value; + List out_rows = _evaluate_func.Invoke(g_rows); + if (out_rows != null && out_rows.Count > 0) { + rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema); + } + rv.addRows(out_rows); + } + return rv; + } + public string[] get_group_by_cols() { + return _groupby_cols_func(); + } + + } private ViewStream _main_stream; private INonIncGroupByEvaluator _groupby_func; private BlueRowHasher _row_hasher; @@ -15,24 +39,29 @@ namespace BCG private Stopwatch _timer = new(); private int _n_total_fires = 0, _n_expensive_fires = 0; - public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) - { + public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) { _main_stream = main_stream; _groupby_func = groupby_func; _groupby_cols = _groupby_func.get_group_by_cols(); _row_hasher = new BlueRowHasher(_groupby_cols.Length); } + public NonIncGroupByStream(ViewStream main_stream, Func groupby_cols_func, + Func, List> evaluate_func) + : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { } + + protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; } protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; } - private void _main_stream_onchange(object nv) - { + private void _main_stream_onchange(object nv) { DataPacket dp = nv as DataPacket; ///grouping together _timer.Restart(); - IDictionary> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher); + IDictionary> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ? + new Dictionary>() { { 0, dp.newRows.to_list() } } : + BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher); _timer.Stop(); var groupElapsedMs = _timer.ElapsedMilliseconds; if (groupElapsedMs >= 50) { @@ -44,14 +73,13 @@ namespace BCG IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows); _timer.Stop(); var applyElapsedMs = _timer.ElapsedMilliseconds; - _n_total_fires++; - if (applyElapsedMs >= 50) - { + _n_total_fires++; + if (applyElapsedMs >= 50) { _n_expensive_fires++; Log.DataFlow.info( $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}"); - } - myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows)); + } + myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows)); } } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs index 4ce9eaa4e..ef5ad6180 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs @@ -5,11 +5,10 @@ using mbts; using System; using System.Collections.Generic; using System.Diagnostics; +using System.Security.Cryptography; -namespace BCG -{ - public class NonIncJoinBookkeeper : IDisposable - { +namespace BCG { + public class NonIncJoinBookkeeper : IDisposable { public Action on_new_data; private readonly NonIncJoinSpec _join_spec; private readonly int _join_position; @@ -26,41 +25,38 @@ namespace BCG private int[] _join_col_indices = null, _data_col_indices = null; - - public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) - { + + public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) { _join_spec = the_join_spec; _join_position = join_position; _start_index_in_output = start_index_in_output; _row_hasher = row_hasher; _is_left_most = _join_position == 0; _is_tag_along = _join_spec.join_col_names.Length == 0; + } + + public void init() { _join_spec.stream.onChange += _on_stream_cb; } - public void _on_stream_cb(object dp_obj) - { + public void _on_stream_cb(object dp_obj) { + Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}"); var timer = Stopwatch.StartNew(); DataPacket dp = (DataPacket)dp_obj; - if (_schema == null) - { + if (_schema == null) { _schema = dp.schema; _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names); _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names); } - if (_is_left_most) - { + if (_is_left_most) { _row_list.Clear(); foreach (IBlueRow r in dp.newRows) - _row_list.Add(new Pair(_row_hasher.get_hash(r, _join_col_indices), r)); - } - else - { + _row_list.Add(new Pair(_row_hasher.get_hash(r, _join_col_indices), r)); + } else { if (_is_tag_along) _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0); - else - { + else { _row_map.Clear(); foreach (IBlueRow r in dp.newRows) _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r; @@ -79,31 +75,25 @@ namespace BCG public bool is_ready { get { return _schema != null; } } - public void constribute_cols_to_schema(List cis) - { + public void constribute_cols_to_schema(List cis) { for (int i = 0; i < _join_spec.data_col_names.Length; i++) cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType)); } public List> get_list() { return _row_list; } - public void copy_master_row_to_output_row(int row_index, object[] output_row) - { + public void copy_master_row_to_output_row(int row_index, object[] output_row) { IBlueRow this_row = _row_list[row_index].second; BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output); } - public void copy_matching_row_to_output_row(int row_hash, object[] output_row) - { - if (_is_tag_along) - { + public void copy_matching_row_to_output_row(int row_hash, object[] output_row) { + if (_is_tag_along) { if (_solitary_row != null) BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output); else copy_dbnull_to_output_row(output_row); - } - else - { + } else { IBlueRow this_row = null; if (_row_map.TryGetValue(row_hash, out this_row)) BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output); @@ -112,8 +102,7 @@ namespace BCG } } - private void copy_dbnull_to_output_row(object[] output_row) - { + private void copy_dbnull_to_output_row(object[] output_row) { for (int i = 0; i < _data_col_indices.Length; i++) output_row[i + _start_index_in_output] = DBNull.Value; } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs index c6cb15fb6..adea44bac 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs @@ -3,63 +3,58 @@ using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; + using mbts; -namespace BCG -{ - public class NonIncJoinSpec - { +namespace BCG { + public class NonIncJoinSpec { public ViewStream stream; public string[] join_col_names, data_col_names, projected_col_names; public NonIncJoinSpec() { } public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names) - : this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names) - { } + : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { } - public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) - { + public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) { stream = the_stream; join_col_names = the_join_col_names; data_col_names = the_data_col_names; projected_col_names = the_projected_col_names; } - public static NonIncJoinSpec[] from_triplet(params Triplet[] join_specs) - { + public static NonIncJoinSpec[] from_triplet(params Triplet[] join_specs) { List rv = new List(); - foreach (Triplet r in join_specs) - { - string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' '); + foreach (Triplet r in join_specs) { + string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' '); string[] visible_cols = r.third.Split(' '); List data_col_names = new List(), mapped_col_names = new List(); - foreach (string s in visible_cols) - { + foreach (string s in visible_cols) { string[] toks = s.Split(':'); - if (toks.Length == 1) - { + if (toks.Length == 1) { data_col_names.Add(toks[0]); mapped_col_names.Add(toks[0]); - } - else if (toks.Length == 2) - { + } else if (toks.Length == 2) { data_col_names.Add(toks[0]); mapped_col_names.Add(toks[1]); } } - rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray())); + rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray())); } - return rv.ToArray (); + return rv.ToArray(); + } + + public override string ToString() { + Func arr_txt = (a) => { + string rv = ""; + foreach (string s in a) { + rv += "{s} "; + } + rv = rv.Trim(); + return rv; + }; + string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}"; + return rv; } } - - //[DataStreamUserRespresentation("ninctest")] - //public class NonIncJoinTestStream : ViewStream - //{ - // public NonIncJoinTestStream (string key_cols, string data_cols) - // { - // NonIncJoinSpec.from_triplet(new Triplet(null, key_cols, data_cols)); - // } - //} } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs index 0a341413e..5c6be1a14 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs @@ -6,11 +6,9 @@ using System; using System.Collections.Generic; using System.Diagnostics; -namespace BCG -{ +namespace BCG { [DataStreamUserRespresentation("fastJoin")] - public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker - { + public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker { private readonly NonIncJoinSpec[] _join_specs; private readonly INonIncJoinInlineMapper _output_mapper; private readonly BlueRowHasher _row_hasher; @@ -28,24 +26,19 @@ namespace BCG { new Triplet (left, left_join_cols, left_proj), new Triplet (right, right_join_cols, right_proj) - }) - { } + }) { } public NonIncJoinStream(params Triplet[] join_specs) - : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) - { } + : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { } public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms, params Triplet[] join_specs) - : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) - { } + : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { } public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) - : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) - { } + : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { } - public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) - { + public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) { _join_specs = join_specs; _output_mapper = output_mapper; _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length); @@ -53,12 +46,11 @@ namespace BCG _vsd = vsd; } - protected override void myOnFirstSubscriber() - { + protected override void myOnFirstSubscriber() { int data_col_no = 1; - for (int jp = 0; jp < _join_specs.Length; jp++) - { + for (int jp = 0; jp < _join_specs.Length; jp++) { NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher); + Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}"); data_col_no += _join_specs[jp].data_col_names.Length; s.on_new_data += _stream_callback; _book_keeping_data.Add(s); @@ -66,10 +58,11 @@ namespace BCG if (_output_mapper != null) _output_mapper.onMapperChange += _on_mapper_change; _vsd.addWorker(this); + foreach (var bkd in _book_keeping_data) + bkd.init(); } - protected override void myOnLastSubscriber() - { + protected override void myOnLastSubscriber() { foreach (NonIncJoinBookkeeper d in _book_keeping_data) d.Dispose(); if (_output_mapper != null) @@ -77,30 +70,26 @@ namespace BCG _vsd.removeWorker(this); } - private void _stream_callback() - { + private void _stream_callback() { + Log.Gen.info("NonIncJoinStream._stream_callback received callback"); _is_dirty = true; doWork(); } - private void _on_mapper_change() - { + private void _on_mapper_change() { _is_dirty = true; doWork(); } - private bool _is_ready() - { + private bool _is_ready() { bool rv = true; foreach (NonIncJoinBookkeeper d in _book_keeping_data) rv &= d.is_ready; return rv; } - private void _construct_schema() - { - if (_output_schema == null) - { + private void _construct_schema() { + if (_output_schema == null) { List cis = new List(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }); foreach (NonIncJoinBookkeeper d in _book_keeping_data) d.constribute_cols_to_schema(cis); @@ -109,27 +98,23 @@ namespace BCG } private int _n_total_fires = 0, _n_expensive_fires = 0; - private void _form_and_fire() - { - if (_is_ready()) - { + private void _form_and_fire() { + Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}"); + if (_is_ready()) { _construct_schema(); var timer = Stopwatch.StartNew(); IList> left_row_list = _book_keeping_data[0].get_list(); List output_rows = new List(); - for (int i = 0; i < left_row_list.Count; i++) - { + for (int i = 0; i < left_row_list.Count; i++) { object[] or = new object[_output_schema.columnInfo.Length]; or[0] = i; output_rows.Add(or); } - for (int ji = 0; ji < _book_keeping_data.Count; ji++) - { + for (int ji = 0; ji < _book_keeping_data.Count; ji++) { NonIncJoinBookkeeper bk_data = _book_keeping_data[ji]; - for (int i = 0; i < left_row_list.Count; i++) - { + for (int i = 0; i < left_row_list.Count; i++) { if (ji == 0) bk_data.copy_master_row_to_output_row(i, output_rows[i]); else @@ -148,10 +133,8 @@ namespace BCG } } - private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) - { - if (_output_mapper != null) - { + private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) { + if (_output_mapper != null) { IBlueRowCollection output_rows; var timer = Stopwatch.StartNew(); output_rows = _output_mapper.map_rows(unmapped_rows); @@ -160,17 +143,14 @@ namespace BCG Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}"); } return output_rows; - } - else - { + } else { return unmapped_rows; } } - public void doWork() - { - if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) - { + public void doWork() { + Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}"); + if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) { _is_dirty = false; _last_fire_time = DateTime.Now; _form_and_fire(); diff --git a/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs b/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs index 296154a75..9fe3d6198 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs @@ -2,9 +2,13 @@ using System; using System.Collections; using System.Collections.Generic; +using Amazon.Runtime.Internal.Util; + using BCG; using BCG.Utilities; +using bts; + namespace BCG { public class LiteralStream : ViewStream { @@ -22,6 +26,34 @@ namespace BCG { return rv; } + //literal("table1", "x:123 y:1.001 z:abc;x:123 y:1.001 z:abc") + public LiteralStream(string table_name, string rows) { + Func get_row = (json_row, id) => { + Dictionary map = new Dictionary(); + string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries); + //Logger.assert(col_vals.Length == 2); + foreach (string t in toks) { + var col_vals = t.Split(':', StringSplitOptions.TrimEntries); + string col = col_vals[0]; + string val = col_vals[1]; + map.Add(col, val); + } + map["id"] = id; + return new BlueRow(map); + }; + string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries); + BlueTable table = null; + foreach (string json_string in row_arr) { + IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count); + get_row(json_string, table == null ? 0 : table.Count); + table = table ?? new BlueTable(r.schema); + table.addRow(r); + } + ViewStream str = new BlueTableStream(table); + str = new SchemaGuesserStream(str); + mySetRealSource(str); + } + public LiteralStream(string tableName, string schema, string value) { string[] columns = schema.Split(' '); @@ -55,7 +87,7 @@ namespace BCG { mySetRealSource(new BlueRowCollectionStaticStream(t)); } - public LiteralStream(string schema, string value) : this("literal", schema, value) { } + // public LiteralStream(string schema, string value) : this("literal", schema, value) { } // public LiteralStream(string value) { diff --git a/src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs b/src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs new file mode 100644 index 000000000..5a360c65d --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs @@ -0,0 +1,72 @@ +using BCG; + +using MongoDB.Driver.Core.WireProtocol.Messages; + +using System; +using System.Collections.Generic; +using System.Linq; + +// the first arg is all the rows, the second arg is the index of the row that needs mapping, and +// the func returns the mapped row (ie mapped version of the index row) +using GlobalMap = System.Func, uint, BCG.IBlueRow>; + +namespace bts { + // In GlobalMap, each row is mapped with the full view of all rows, ie the map functions for a row + // can use the state fo all rows. + public class GlobalMapStream : ViewStream { + private ViewStream _base_stream; + private readonly string _sort_col_name, _group_by_col; + private GlobalMap _map_func; + public GlobalMapStream(ViewStream base_stream, string sort_col_name, string group_by_col, GlobalMap map_func) { + _base_stream = base_stream; + _sort_col_name = sort_col_name; + _group_by_col = group_by_col; + _map_func = map_func; + } + + public GlobalMapStream(ViewStream base_stream, string sort_col_name, GlobalMap map_func) : + this(base_stream, sort_col_name, null, map_func) { + } + + protected override void myOnFirstSubscriber() { + _base_stream.onChange += _handle_stream_update; + } + + protected override void myOnLastSubscriber() { + _base_stream.onChange -= _handle_stream_update; + } + + void _handle_stream_update(object dp_obj) { + DataPacket dp = dp_obj as DataPacket; + IDictionary> groups = !String.IsNullOrEmpty(_group_by_col) ? + BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col }) + : + new Dictionary>() { { 0, dp.newRows.to_list() } }; + BlueTable out_table = null; + foreach (var kv in groups) { + List group_rows = kv.Value; + group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col_name)); + uint i = 0; + foreach (IBlueRow r in group_rows) { + IBlueRow nr = _map_func(group_rows, i++); + out_table = out_table ?? new BlueTable(nr.schema); + out_table.addRow(nr); + } + } + + + // List rows = dp.newRows.to_ienum().ToList(); + // rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r1, r2, _sort_col_name)); + // uint i = 0; + + // BlueTable out_table = null; + // foreach (IBlueRow r in rows) { + // IBlueRow nr = _map_func(rows, i++); + // out_table = out_table ?? new BlueTable(nr.schema); + // out_table.addRow(nr); + // } + DataPacket dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket); + myFireEvent(dp_out); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs index 8cfce3ba6..e4cc58c44 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs @@ -41,7 +41,6 @@ namespace BCG { } _table.addRows(added_rows); myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema))); - } } diff --git a/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs new file mode 100644 index 000000000..407ee4c24 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs @@ -0,0 +1,65 @@ +using System; +using System.Data; +using System.Data.SQLite; + +using BCG; + +namespace bts { + public static class SQLiteHelper { + public static DataTable ExecuteSelectQuery(string connection_string, string query) { + // Create a new DataTable to store the results + DataTable dataTable = new DataTable(); + + // Create and open a SQLite connection using the provided connection string + using (SQLiteConnection connection = new SQLiteConnection(connection_string)) { + try { + connection.Open(); + + // Create a SQLite command with the provided query + using (SQLiteCommand command = new SQLiteCommand(query, connection)) { + // Use SQLiteDataAdapter to fill the DataTable + using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command)) { + adapter.Fill(dataTable); + } + } + } catch (Exception ex) { + Console.WriteLine("An error occurred: " + ex.Message); + } + } + + // Return the populated DataTable + return dataTable; + } + } + + [DataStreamUserRespresentation("SQLite")] + public class SQLiteStream : ViewStream { + private readonly string _connection_string, _query; + public SQLiteStream(string connection_string, string query) { + _connection_string = connection_string; + _query = query; + } + + + private void _read_from_sqlite_nd_fire() { + DataTable dt = SQLiteHelper.ExecuteSelectQuery(_connection_string, _query); + IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt); + + DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket); + myFireEvent(dp); + } + + protected override void myOnFirstSubscriber() { + _read_from_sqlite_nd_fire(); + } + + protected override void myOnLastSubscriber() { + // this value will be cleared by the base class + // but in this particular stream this behaviour + // is not usefull since any new subscribers + // would force the query to be recreated + //myLastDP = myLastValue; + } + + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/StaticCollectionStream/BlueTableStream.cs b/src/DataStreamSuite/DataStream/Primitives/StaticCollectionStream/BlueTableStream.cs index d1b7c519c..c2a458a07 100644 --- a/src/DataStreamSuite/DataStream/Primitives/StaticCollectionStream/BlueTableStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/StaticCollectionStream/BlueTableStream.cs @@ -1,31 +1,41 @@ using System; using System.Collections; using System.Reflection; + using BCG.Utilities; -namespace BCG -{ - [DataStreamUserRespresentation (false)] - public class BlueRowCollectionStaticStream : ViewStream - { - public BlueRowCollectionStaticStream () {} - - public BlueRowCollectionStaticStream (IBlueRowCollection rows) {loadCollection (rows);} - - public void loadCollection (IBlueRowCollection rows) - { - myFireEvent (new DataPacket (rows, rows, myLastValue != null ? - new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema))); +namespace BCG { + // if a table needs to be fired, it can be wrapped around this + public class BlueTableStream : ViewStream { + private IBlueRowCollection _rows; + public BlueTableStream(IBlueRowCollection rows) { + _rows = rows; } - public void crush_with_single_row(IBlueRow r) - { + protected override void myOnFirstSubscriber() { + var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket); + myFireEvent(dp_out); + } + } + + [DataStreamUserRespresentation(false)] + public class BlueRowCollectionStaticStream : ViewStream { + public BlueRowCollectionStaticStream() { } + + public BlueRowCollectionStaticStream(IBlueRowCollection rows) { loadCollection(rows); } + + public void loadCollection(IBlueRowCollection rows) { + myFireEvent(new DataPacket(rows, rows, myLastValue != null ? + new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable(rows.schema))); + } + + public void crush_with_single_row(IBlueRow r) { BlueTable t = new BlueTable(r.schema); t.addRow(r); loadCollection(t); } - public void loadPacket (DataPacket dp) {myFireEvent (dp);} + public void loadPacket(DataPacket dp) { myFireEvent(dp); } /// /// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc @@ -35,17 +45,16 @@ namespace BCG /// /// /// - public static IBlueRowCollection statelessOp (IBlueRowCollection rows, Type streamType, params object[] args) - { - ArrayList totArgs = new ArrayList (args); - BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream (); - totArgs.Insert (0, v1); - ViewStream retVal = Activator.CreateInstance (streamType, totArgs.ToArray()) as ViewStream; - v1.loadCollection (rows); + public static IBlueRowCollection statelessOp(IBlueRowCollection rows, Type streamType, params object[] args) { + ArrayList totArgs = new ArrayList(args); + BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream(); + totArgs.Insert(0, v1); + ViewStream retVal = Activator.CreateInstance(streamType, totArgs.ToArray()) as ViewStream; + v1.loadCollection(rows); return retVal.rows; } } - + } diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/DateTimeExpanstionStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/DateTimeExpanstionStream.cs new file mode 100644 index 000000000..03a4ec900 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Time/DateTimeExpanstionStream.cs @@ -0,0 +1,34 @@ +namespace bts.SilverPine { + using bts; + + using BCG; + + using System; + + public class DateTimeExpanstionStream : ViewStream { + public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) { + Func get_schema = (s) => { + IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date"); + rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day"); + rv = rv.add_column(System.TypeCode.String, $"{inp_col}_month"); + rv = rv.add_column(System.TypeCode.String, $"{inp_col}_hour"); + rv = rv.add_column(System.TypeCode.String, $"{inp_col}_min"); + return rv; + }; + ViewStream rs = new MapStream(parent_stream, + s => get_schema(s), + r => { + DateTime dt = r.get_datetime(inp_col); + IBlueTableSchema s = get_schema(r.schema); + MutableBlueRow rv = new MutableBlueRow(s, r); + rv.setValue($"{inp_col}_date", dt.Date); + rv.setValue($"{inp_col}_day", dt.DayOfWeek.ToString()); + rv.setValue($"{inp_col}_month", dt.Date.Month.ToString()); + rv.setValue($"{inp_col}_hour", dt.Hour); + rv.setValue($"{inp_col}_min", dt.Minute); + return rv; + }); + mySetRealSource(rs); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs b/src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs new file mode 100644 index 000000000..90e660f22 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs @@ -0,0 +1,111 @@ +namespace bts.SilverPine { + using BCG; + + using bts; + using bts.collections; + + using System; + using System.CodeDom; + using System.Collections.Generic; + using System.Linq; + + // Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows + // so: + // f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2] + [DataStreamUserRespresentation("ColDiff")] + public class ColumnDifferenceStream : ViewStream { + private readonly ViewStream _base_stream; + private readonly FilterStream.OnFilter _f1, _f2; + private readonly string _c1, _c2; + + public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) { + _base_stream = base_stream; + _f1 = f1; + _f2 = f2; + _c1 = col1; + _c2 = col2; + } + + public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) { + FilterStream.OnFilter f1_del = r => { + BooleanExpressionFilter rv = new BooleanExpressionFilter(f1); + return rv.filter(r); + }; + FilterStream.OnFilter f2_del = r => { + BooleanExpressionFilter rv = new BooleanExpressionFilter(f2); + return rv.filter(r); + }; + ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2); + mySetRealSource(rs); + } + + // for each row, does a row[new_ol_name] = row[col1] - rows[row_ffset][col2] + // we should generlize this for any math operation + public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) { + // use GlobalMapStream + string diff_col_perc_name = $"{diff_col_name}_percentage"; + System.Func, uint, BCG.IBlueRow> g_map_func = (rows, idx) => { + int minuend_idx = Convert.ToInt32(idx); + var inp_schema = rows[0].schema; + var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name); + out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name); + // minuend - subtrahend + IBlueRow minuend_row = rows.ElementAt(minuend_idx); + MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row); + int subtrahend_index = minuend_idx + row_offset; + double diff_val = Double.NaN, diff_val_perc = Double.NaN; + if (subtrahend_index >= 0) { + try { + IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index); + diff_val = minuend_row.get(col1) - subtrahend_row.get(col2); + diff_val_perc = 100.0 * diff_val / subtrahend_row.get(col1); + } catch (Exception ex) { + Log.assert(false, ex.Message); + } + } + rv.setValue(diff_col_name, diff_val); + rv.setValue(diff_col_perc_name, diff_val_perc); + return rv; + }; + ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func); + if (optional_perc_diff_filter != 0) { + rs = new FilterStream(rs, r => { + double diff_val_perc = r.get(diff_col_perc_name); + double abs_diff_val_perc = Math.Abs(diff_val_perc); + return abs_diff_val_perc >= optional_perc_diff_filter; + }); + } + mySetRealSource(rs); + } + + protected override void myOnFirstSubscriber() { + _base_stream.onChange += _handle_base_stream_update; + } + + protected override void myOnLastSubscriber() { + _base_stream.onChange -= _handle_base_stream_update; + } + + private void _handle_base_stream_update(object dp_obj) { + // could have been done with filter and cartesian prod, and then math + DataPacket dp = dp_obj as DataPacket; + List rows = dp.newRows.to_list(); + var f1_rows = rows.Where(r => _f1(r)); + Log.assert(f1_rows.Count() == 1); + var f2_rows = rows.Where(r => _f2(r)); + Log.assert(f2_rows.Count() == 1); + IBlueRow f1_r = f1_rows.ElementAt(0); + IBlueRow f2_r = f2_rows.ElementAt(0); + double diff = f1_r.get(_c1) - f2_r.get(_c2); + IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff")); + BlueTable out_table = new BlueTable(out_schema); + MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r); + out_row.setValue("diff", diff); + out_table.addRow(out_row); + DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket); + myFireEvent(dp_out); + + } + } + +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnEvolutionStream.cs b/src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnEvolutionStream.cs new file mode 100644 index 000000000..a282a04d4 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnEvolutionStream.cs @@ -0,0 +1,51 @@ +namespace bts.SilverPine { + using BCG; + + using bts; + + using System.Collections.Generic; + using System.Linq; + + // Given a sort order, an optional group by col, it observes the values of a column + // within that group (or the whole table, if grouping not specified) and returns all rows + // where the value changes, ie row[current][col] != row[current-1][col] + // TODO: a more general way to express a filter where row[current][col] != row[previous][col], + // or even expand it to row[current][col_a] - row[previous][col_b] < 0.5 + [DataStreamUserRespresentation("ColEvol")] + public class ColumnEvolutionStream : ViewStream { + private readonly ViewStream _base_stream; + private readonly string _sort_col; + private readonly string _evol_col; + public ColumnEvolutionStream(ViewStream base_stream, string sort_col, string evolution_col) { + _base_stream = base_stream; + _sort_col = sort_col; + _evol_col = evolution_col; + } + + protected override void myOnFirstSubscriber() { + _base_stream.onChange += _handle_base_stream_update; + } + + protected override void myOnLastSubscriber() { + base.myOnLastSubscriber(); + } + + private void _handle_base_stream_update(object dp_obj) { + DataPacket dp = dp_obj as DataPacket; + List rows = dp.newRows.to_list(); + rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col)); + BlueTable rv = new BlueTable(dp.schema); + IBlueRow prev_row = rows.ElementAt(0); + foreach (IBlueRow r in rows) { + var cr = BlueDataUtility.compare_rows(r, prev_row, _evol_col); + if (cr != 0) { + rv.addRow(r); + } + prev_row = r; + } + DataPacket dp_out = BlueDataUtility.build_dp(rv, myLastValue as DataPacket); + myFireEvent(dp_out); + } + } + +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs index aa09e83bb..d967c1409 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs @@ -15,6 +15,8 @@ using mbts; using BCG.Extensions; using System.Text.Json; using System.Text.Json.Nodes; +using Newtonsoft.Json; + namespace BCG { public static class BlueDataUtility { @@ -49,6 +51,17 @@ namespace BCG { rv.Add(r.get(col)); return rv; } + public static IEnumerable get_colvalues(IEnumerable rows, string col, int start_index, int count) { + List rv = new List(); + int i = 0; + int n_copied = 0; + foreach (IBlueRow r in rows) + if (i++ >= start_index && n_copied < count) { + rv.Add(r.get(col)); + n_copied++; + } + return rv; + } public static IDictionary> group_by(IBlueRowCollection rows, string gp_col) { Dictionary> rv = new Dictionary>(); @@ -112,13 +125,17 @@ namespace BCG { } } - public static IEnumerable to_ienum(this IBlueRowCollection rows) { + public static List to_list(this IBlueRowCollection rows) { List rv = new List(); foreach (IBlueRow r in rows) rv.Add(r); return rv; } + public static IEnumerable to_ienum(this IBlueRowCollection rows) { + return rows.to_list(); + } + public static IDictionary to_map(this IBlueRowCollection rows) { Dictionary rv = new Dictionary(); foreach (IBlueRow r in rows) { @@ -135,6 +152,11 @@ namespace BCG { return new BlueTableSchema(input_schema.tableName, cis.ToArray()); } + public static IBlueTableSchema add_column(this IBlueTableSchema input_schema, TypeCode type_arg, string col_name) { + Tuple tup = new Tuple(_typecode_to_type[type_arg], col_name); + return add_columns(input_schema, tup); + } + public static IBlueRowCollection to_collection(this IBlueRow r) { if (r == null) return null; @@ -191,6 +213,18 @@ namespace BCG { #region json + public static Dictionary convert_json_to_dictionary(string json_string) { + Dictionary dictionary = JsonConvert.DeserializeObject>(json_string); + return dictionary; + } + public static IBlueRow convert_json_to_row(string json_string, int id = 0) { + Dictionary dictionary = JsonConvert.DeserializeObject>(json_string); + if (!dictionary.ContainsKey("id")) { + dictionary.Add("id", id); + } + return new BlueRow(dictionary); + } + public static List json_to_json_node_list(string jsonString) { var jsonNodeList = new List(); bool noe = String.IsNullOrEmpty(jsonString); @@ -209,7 +243,7 @@ namespace BCG { foreach (JsonNode jsonNode in jsonArray) { jsonNodeList.Add(jsonNode); } - } catch (JsonException ex) { + } catch (System.Text.Json.JsonException ex) { Console.WriteLine($"Invalid JSON: {ex.Message}"); } @@ -486,7 +520,16 @@ namespace BCG { ColumnInfo[] cis = new ColumnInfo[table.Columns.Count]; foreach (DataColumn col in table.Columns) cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType); - return new BlueTableSchema(cis, table.TableName); + IBlueTableSchema rv = new BlueTableSchema(cis, table.TableName); + rv = rv.add_id_col_if_missing(); + return rv; + } + public static IBlueTableSchema add_id_col_if_missing(this IBlueTableSchema schema) { + IBlueTableSchema rv = schema; + if (!schema.columnNames.Contains("id")) { + rv = schema.add_columns(Tuple.Create(typeof(int), "id")); + } + return rv; } /// @@ -744,10 +787,16 @@ namespace BCG { } public static IBlueRowCollection toIBlueRowCollection(DataTable dt) { - BlueTable retVal = new BlueTable(getSchema(dt)); - foreach (DataRow dr in dt.Rows) - retVal.addRow(dr); - return retVal; + IBlueTableSchema schema = getSchema(dt); + BlueTable rv = new BlueTable(schema); + foreach (DataRow dr in dt.Rows) { + MutableBlueRow nr = new MutableBlueRow(schema); + foreach (DataColumn col in dr.Table.Columns) + nr.setValue(col.ColumnName, dr[col.ColumnName]); + nr.setValue("id", rv.Count); + rv.addRow(nr); + } + return rv; } public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) { @@ -819,20 +868,20 @@ namespace BCG { /// /// /// - /// + /// /// public static int compare_rows(IBlueRow r1, IBlueRow r2, - params string[] colNames) { - int retVal = 0; - foreach (string colName in colNames) { - object val1 = r1[colName]; - object val2 = r2[colName]; - retVal = _compare_row_cols(val1, val2); - if (retVal != 0) + params string[] col_names) { + int comp_res = 0; + foreach (string cn in col_names) { + object val1 = r1[cn]; + object val2 = r2[cn]; + comp_res = _compare_row_cols(val1, val2); + if (comp_res != 0) break; } - return retVal; + return comp_res; } private static int _compare_row_cols(object v1, object v2) { @@ -1628,6 +1677,13 @@ namespace BCG { return sb.ToString(); } + public static string id_string(IBlueRowCollection table) { + string rv = String.Empty; + foreach (IBlueRow r in table) + rv += $"{r["id"]} "; + return rv; + } + public static IDictionary> group_together(IBlueRowCollection rows, string[] group_by_col_names) { BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length); return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher); @@ -1776,6 +1832,18 @@ namespace BCG { #endregion } + public static IBlueRow find_max(IEnumerable rows, params string[] cols_to_compare) { + if (rows == null || rows.Count() == 0) + return null; + IBlueRow max_row = rows.ElementAt(0); + foreach (IBlueRow r in rows) { + int c = compare_rows(r, max_row, cols_to_compare); + if (c > 0) + max_row = r; + } + return max_row; + } + public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary sort_map) { foreach (KeyValuePair p in sort_map) { int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value); diff --git a/src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj b/src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj index 95d4024d2..88cc3561c 100644 --- a/src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj +++ b/src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj @@ -8,6 +8,7 @@ + diff --git a/src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs b/src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs new file mode 100644 index 000000000..658a67b18 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs @@ -0,0 +1,181 @@ +using bts.SilverPine; + +namespace bts.SilverPine { + using bts.api; + using bts.spark; + + using System; + + using ZstdSharp.Unsafe; + + public static class ContractUtils { + #region Parsing + public static FutureContract ParseFutureContract(string contract_txt, Exchange exchange = Exchange.CME) { + // Validate the future string length + if (!contract_txt.All(c => Char.IsLetterOrDigit(c))) + return null; + if (contract_txt.Length < 4) { + return null; + } + + // Extract the symbol and the expiration month/year codes + string symbol = contract_txt.Substring(0, contract_txt.Length - 2); + char monthCode = contract_txt[contract_txt.Length - 2]; + char yearCode = contract_txt[contract_txt.Length - 1]; + + // Map month code to month, return null if the month code is invalid + int month = monthCode switch { + 'F' => 1, // January + 'G' => 2, // February + 'H' => 3, // March + 'J' => 4, // April + 'K' => 5, // May + 'M' => 6, // June + 'N' => 7, // July + 'Q' => 8, // August + 'U' => 9, // September + 'V' => 10, // October + 'X' => 11, // November + 'Z' => 12, // December + _ => 0 // Return 0 for invalid month code + }; + + if (month == 0) { + return null; // Invalid month code + } + + // Calculate the year based on the current decade + int currentYear = DateTime.Now.Year; + int decadeStart = currentYear / 10 * 10; + int year = decadeStart + (yearCode - '0'); + + // If the year is in the past, assume the next decade + if (year < currentYear) { + year += 10; + } + + // Create the FutureContract instance and set its properties + DateTime expiration = new DateTime(year, month, 1); + var futureContract = new FutureContract { + symbol = symbol, + expiration = expiration, + exchange = exchange + }; + + return futureContract; + } + // Method to parse a futures spread, e.g. "CLX4-CLZ4" + public static FutureSpreadContract ParseFutureSpreadContract(string spreadString, Exchange exchange = Exchange.CME) { + // Split the string by '-' to handle each leg + var spreadParts = spreadString.Split('-'); + if (spreadParts.Length < 2) { + return null; // Not a valid spread format + } + + var spreadContract = new FutureSpreadContract { legs = new List>() }; + + foreach (var part in spreadParts) { + // Split each leg by '*', assuming the format is weight*future or just future + string[] legParts = part.Split('*'); + double weight = 1.0; + string futurePart; + + if (legParts.Length == 2) { + if (!double.TryParse(legParts[0], out weight)) { + return null; // Invalid weight, parsing failed + } + futurePart = legParts[1]; + } else if (legParts.Length == 1) { + futurePart = legParts[0]; + } else { + return null; // Invalid format, too many '*' parts + } + + // Parse the future contract + var futureContract = ParseFutureContract(futurePart, exchange); + if (futureContract == null) { + return null; // Failed to parse the future part + } + + // Add the leg to the spread contract + spreadContract.legs.Add(new Tuple(weight, futureContract)); + } + + return spreadContract; + } + + public static ContractType DetermineFutureOrSpread(string contract) { + if (contract.Contains("-")) { + return ContractType.FutureSpread; + } else { + return ContractType.Future; + } + } + #endregion + + public static ContractType get_type(string contract_txt) { + FutureContract fc = ParseFutureContract(contract_txt); + if (fc != null) { + return ContractType.Future; + } + FutureSpreadContract sc = ParseFutureSpreadContract(contract_txt); + if (sc != null) { + return ContractType.FutureSpread; + } + Log.assert(false); + return bts.SilverPine.ContractUtils.ContractType.Spot; + } + + public static ContractCategory get_category(string symbol) { + var tp = get_symbol_category(symbol); + return tp.Item1; + } + + public static Tuple get_symbol_category(string symbol) { + ContractUtils.FutureContract fc = ParseFutureContract(symbol); + if (fc != null) { + return Tuple.Create(ContractCategory.FutureString, fc.symbol); + } + var sc = ParseFutureSpreadContract(symbol); + if (sc != null) { + return Tuple.Create(ContractCategory.FutureString, sc.symbol); + } + return Tuple.Create(ContractCategory.Symbol, symbol); + } + + + + public enum ContractType { + Spot, + Option, + Future, + FutureSpread, + OptionSpread + } + public enum ContractCategory { + Symbol = 0, + FutureString, + SpreadString + } + public enum CallPut { + Call, + Put + } + public enum Exchange { + CME + } + public class FutureContract { + public string symbol; + public DateTime expiration; + public Exchange exchange; + } + public class FutureSpreadContract { + public List> legs; + public string symbol { + get { + return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol; + } + } + } + } +} diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/AbsoluteVolumeStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/AbsoluteVolumeStream.cs new file mode 100644 index 000000000..d5d5e8d2d --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/AbsoluteVolumeStream.cs @@ -0,0 +1,47 @@ +namespace bts.SilverPine { + using BCG; + + using ZstdSharp.Unsafe; + + [DataStreamUserRespresentation("AbsVolume")] + // Calculates the volume within a time window (this does not do rates, like highest daily volume etc, + // but with some thought can perhaps be extended) + public class AbsoluteVolumeStream : ViewStream { + // TODO, abstract this out + // adds contract_type to OHLCV stream + private static ViewStream _get_ohlcv_steam(string symbol, DateTime d1, DateTime d2) { + var ohlcv_params = OHLCVStream.get_params(d1, d2); + ViewStream rs = new OHLCVStream(symbol, ohlcv_params); + Func create_schema = + s => BlueDataUtility.add_columns(s, Tuple.Create(typeof(string), "contract_type")); + rs = new SuperMapStream(rs, + s => create_schema(s), + r => { + IBlueTableSchema out_schema = create_schema(r.schema); + string contract_txt = r.get("contract"); + var contr_type = ContractUtils.get_type(contract_txt); + MutableBlueRow out_row = new MutableBlueRow(out_schema, r); + out_row.setValue("contract_type", contr_type.ToString()); + return new IBlueRow[] { out_row }; + }); + return rs; + } + + public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1) + : this(symbol, ctype, d1, d1) { } + + public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1, DateTime d2) + : this(symbol, Enum.Parse(ctype), d1, d2) { + } + + private AbsoluteVolumeStream(string symbol, ContractUtils.ContractType ctype, DateTime d1, DateTime d2) { + ViewStream ohlcv_str = _get_ohlcv_steam(symbol, d1, d2); + ohlcv_str = new FilterStream(ohlcv_str, + r => r.get_datetime("ts_event") <= d1 && r.get_datetime("ts_event") >= d2); + ohlcv_str = new FilterStream(ohlcv_str, r => String.Compare(ctype.ToString(), r.get("contract_type"), true) == 0); + ViewStream rs = new MaxStream(ohlcv_str, "volume"); + mySetRealSource(rs); + } + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/ChartingHelper.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/ChartingHelper.cs new file mode 100644 index 000000000..34be327d0 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/ChartingHelper.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using bts; + +namespace bts.SilverPine { + public static class ChartingHelper { + public static double calculate_VWAP(List> volumePriceData) { + Log.Gen.info($"ChartingHelper.calculate_VWAP n_vals: {volumePriceData.Count}"); + double totalVolume = 0; + double totalPriceVolume = 0; + + foreach (var data in volumePriceData) { + int volume = data.Item1; // Volume + double price = data.Item2; // Closing price + + // Accumulate total price-volume (price * volume) and total volume + totalPriceVolume += price * volume; + totalVolume += volume; + } + + // VWAP is totalPriceVolume divided by totalVolume + return totalVolume > 0 ? totalPriceVolume / totalVolume : 0; + } + + // Calculate Volatility (Standard Deviation of returns) over a given period + public static double calculate_volatility(IList prices, int period = 252) { + if (prices == null || prices.Count < period + 1) { + throw new ArgumentException("Insufficient data to calculate volatility."); + } + + // Calculate daily returns + var returns = new List(); + for (int i = 1; i < prices.Count; i++) { + double dailyReturn = (prices[i] - prices[i - 1]) / prices[i - 1]; + returns.Add(dailyReturn); + } + + // Get the returns over the desired period + var recentReturns = returns.Skip(returns.Count - period).Take(period).ToList(); + + // Calculate the mean of returns + double meanReturn = recentReturns.Average(); + + // Calculate the variance (sum of squared differences from the mean) + double variance = recentReturns.Sum(r => Math.Pow(r - meanReturn, 2)) / period; + + // Volatility is the square root of variance (standard deviation) + double volatility = Math.Sqrt(variance); + + // Annualize volatility by multiplying by the square root of 252 (number of trading days in a year) + return volatility * Math.Sqrt(252); + } + + // Calculate SMA (Simple Moving Average) for a given period (e.g., 50, 100, or 200) + public static double calculate_SMA(IList prices, int period = 50) { + if (prices == null || prices.Count < period) { + throw new ArgumentException("Insufficient data to calculate SMA."); + } + + // Sum up the prices over the given period + double sum = prices.Skip(prices.Count - period).Take(period).Sum(); + + // Return the average + return sum / period; + } + + public static double calculate_RSI(IList prices, int period = 14) { + if (prices == null || prices.Count < period + 1) { + return Double.NaN; + //throw new ArgumentException("Insufficient data to calculate RSI."); + } + + double gainSum = 0; + double lossSum = 0; + + // Calculate the initial average gain and loss over the first 'period' days + for (int i = 1; i <= period; i++) { + double change = prices[i] - prices[i - 1]; + if (change > 0) { + gainSum += change; + } else { + lossSum -= change; // Subtract since loss is negative + } + } + + double avgGain = gainSum / period; + double avgLoss = lossSum / period; + + // Calculate RSI using smoothed values + for (int i = period + 1; i < prices.Count; i++) { + double change = prices[i] - prices[i - 1]; + if (change > 0) { + avgGain = (avgGain * (period - 1) + change) / period; + avgLoss = (avgLoss * (period - 1)) / period; + } else { + avgGain = (avgGain * (period - 1)) / period; + avgLoss = (avgLoss * (period - 1) - change) / period; + } + } + + // Prevent division by zero in case of no losses + if (avgLoss == 0) { + return 100; + } + + double rs = avgGain / avgLoss; + double rsi = 100 - (100 / (1 + rs)); + + return rsi; + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/DailyChangeStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/DailyChangeStream.cs new file mode 100644 index 000000000..6df0cb841 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/DailyChangeStream.cs @@ -0,0 +1,31 @@ +namespace bts.SilverPine { + using BCG; + + using bts; + + [DataStreamUserRespresentation("DailyChange")] + // takes in the ohlcv stream and computes the closing vs closing difference + public class DailyChange : ViewStream { + public DailyChange(ViewStream ohlc_str) { + GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract", + (rows, idx) => { + double daily_change = Double.NaN; + double daily_change_perc = Double.NaN; + IBlueRow ir = rows.ElementAt((int)idx); + if (idx < rows.Count - 1) { + IBlueRow pr = rows.ElementAt((int)idx + 1); + double close = ir.get("close"), prev_close = pr.get("close"); + daily_change = close - prev_close; + daily_change_perc = 100.0 * daily_change / prev_close; + } + IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "daily_change")); + schema = schema.add_columns(Tuple.Create(typeof(double), "daily_change_perc")); + MutableBlueRow nr = new MutableBlueRow(schema, ir); + nr.setValue("daily_change", daily_change); + nr.setValue("daily_change_perc", daily_change_perc); + return nr; + }); + mySetRealSource(rs); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/EMAStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/EMAStream.cs deleted file mode 100644 index 8303e218e..000000000 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/EMAStream.cs +++ /dev/null @@ -1,54 +0,0 @@ - -using Google.Protobuf.WellKnownTypes; - -using System; -using BCG.Utilities.General; -using System.Data; -using bts.api.domain; - -namespace BCG { - // Stream computes the EMA of a column in any stream. It takes in the base stream, the column name and a smoothing factor - // as args, and create one row with an ema column, which is the running ema of the column provided - [DataStreamUserRespresentation("ema")] - public class EMAStream : ViewStream { - private ViewStream _stream; - private readonly string _col_name; - private EMACalculator _calc; - private readonly IBlueTableSchema _schema = BlueDataUtility.form_schema( - "SMA", Tuple.Create("id", TypeCode.Int32), - Tuple.Create("ema", TypeCode.Double)); - - public EMAStream(ViewStream stream, string col_name, double smoothing_factor) { - _col_name = col_name; - _stream = stream; - _calc = new EMACalculator(smoothing_factor); - } - - protected override void myOnFirstSubscriber() { - _stream.onChange += new ChangeDelegate(_onstream_change); - } - protected override void myOnLastSubscriber() { - _stream.onChange -= new ChangeDelegate(_onstream_change); - } - - - private void _onstream_change(object o) { - - DataPacket dp = o as DataPacket; - foreach (IBlueRow r in dp.addedRows) { - double price = r.get(_col_name); - if (!Double.IsNaN(price)) - _calc.add(price); - } - IBlueRow nr = new BlueRow(_schema, 0, _calc.get_EMA()); - BlueTable state = new BlueTable(_schema); - state.addRow(nr); - DataPacket dp_out = BlueDataUtility.build_dp_inc(state, myLastValue as DataPacket); - myFireEvent(dp_out); - } - } - -} - - - diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RSIStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RSIStream.cs new file mode 100644 index 000000000..e88fc4111 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RSIStream.cs @@ -0,0 +1,26 @@ +namespace bts.SilverPine { + using BCG; + + using bts; + + [DataStreamUserRespresentation("RSI")] + public class RSIStream : ViewStream { + // public RSIStream(ViewStreamDatabase vsd, string symbol, int n_days) { + + // } + + public RSIStream(ViewStream instrument_ohlc_str, int period) { + GlobalMapStream rs = new GlobalMapStream(instrument_ohlc_str, "ts_event", "contract", + (rows, idx) => { + IBlueRow ir = rows.ElementAt((int)idx); + IEnumerable closes = BlueDataUtility.get_colvalues(rows, "close", (int)idx + 1, period + 2); + double rsi = ChartingHelper.calculate_RSI(closes.ToList(), period); + IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "rsi")); + MutableBlueRow nr = new MutableBlueRow(schema, ir); + nr.setValue("rsi", rsi); + return nr; + }); + mySetRealSource(rs); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RealizedVolatilityStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RealizedVolatilityStream.cs new file mode 100644 index 000000000..e69de29bb diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RenkoStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RenkoStream.cs deleted file mode 100644 index c1506c268..000000000 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RenkoStream.cs +++ /dev/null @@ -1,87 +0,0 @@ - -using Google.Protobuf.WellKnownTypes; - -using System; -using BCG.Utilities.General; -using System.Data; -using bts.api.domain; -using System.Reactive.Concurrency; -using mbts.spark; - -namespace BCG { - // Stream computes the EMA of a column in any stream. It takes in the base stream, the column name and a smoothing factor - // as args, and create one row with an ema column, which is the running ema of the column provided - [DataStreamUserRespresentation("renko")] - public class RenkoStream : ViewStream { - private ViewStream _stream; - private readonly string _col_name; - private readonly double _brick_sz; - private RenkoCalculator _calc; - - private BlueTable _table = new BlueTable(BlueDataUtility.form_schema( - "Renko", Tuple.Create("id", TypeCode.Int32), - Tuple.Create("entry_price", TypeCode.Double), - Tuple.Create("current_price", TypeCode.Double), - Tuple.Create("brick_low", TypeCode.Double), - Tuple.Create("brick_high", TypeCode.Double), - Tuple.Create("brick_transition", TypeCode.String), - Tuple.Create("n_updates_within", TypeCode.Int32) - )); - - public RenkoStream(ViewStream stream, string col_name, double brick_sz) { - _col_name = col_name; - _stream = stream; - _brick_sz = brick_sz; - _calc = new RenkoCalculator(brick_sz); - } - - protected override void myOnFirstSubscriber() { - _stream.onChange += _onstream_change; - } - protected override void myOnLastSubscriber() { - _stream.onChange -= new ChangeDelegate(_onstream_change); - } - - - private IBlueRow _remove_row(RenkoBrick brick) { - var row = _table[brick.id]; - _table.deleteRow(row); - return row; - } - - private IBlueRow _create_new_row(RenkoBrick brick) { - return new BlueRow(_table.schema, brick.id, - brick.entry_price, brick.current_price, brick.low, brick.high, - brick.entry.ToString(), brick.n_updates); - } - - private void _onstream_change(object o) { - // what feeds in is a time series - BlueTable deleted_rows = new BlueTable(_table.schema); - BlueTable added_rows = new BlueTable(_table.schema); - DataPacket dp = o as DataPacket; - foreach (IBlueRow r in dp.addedRows) { - double val = r.get(_col_name); - var new_bricks = _calc.update(val); - if (new_bricks.Count() > 0) { - foreach (var nb in new_bricks) { - var new_row = _create_new_row(nb); - added_rows.addRow(new_row); - _table.addRow(new_row); - } - } else { - var existing_brick = _calc.current_brick; - var del_row = _remove_row(existing_brick); - deleted_rows.addRow(del_row); - var new_row = _create_new_row(existing_brick); - added_rows.addRow(new_row); - _table.addRow(new_row); - } - } - myFireEvent(new DataPacket(_table, added_rows, deleted_rows)); - } - } -} - - - diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SMAStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SMAStream.cs index 3bea28f4f..996c7a226 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SMAStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SMAStream.cs @@ -1,54 +1,24 @@ +namespace bts.SilverPine { + using bts; + using BCG; -using Google.Protobuf.WellKnownTypes; - -using System; -using BCG.Utilities.General; -using System.Data; -using bts.api.domain; - -namespace BCG { - // Stream computes the SMA of a column in any stream. It takes in the base stream, the column namer and a smoothing factor - // as args, and create one row with an sma column, which is the running ema of the column provided - [DataStreamUserRespresentation("sma")] - public class SMAStream : ViewStream { - private ViewStream _stream; - private readonly string _col_name; - private Chartutils.SMACalculator _calc; - private readonly IBlueTableSchema _schema = BlueDataUtility.form_schema( - "SMA", Tuple.Create("id", TypeCode.Int32), - Tuple.Create("sma", TypeCode.Double)); - - public SMAStream(ViewStream stream, string col_name, int window_size) { - _col_name = col_name; - _stream = stream; - _calc = new Chartutils.SMACalculator(window_size); - } - - protected override void myOnFirstSubscriber() { - _stream.onChange += new ChangeDelegate(_onstream_change); - } - protected override void myOnLastSubscriber() { - _stream.onChange -= new ChangeDelegate(_onstream_change); - } - - - private void _onstream_change(object o) { - - DataPacket dp = o as DataPacket; - foreach (IBlueRow r in dp.addedRows) { - double price = r.get(_col_name); - if (!Double.IsNaN(price)) - _calc.add(r.get(_col_name)); - } - IBlueRow nr = new BlueRow(_schema, 0, _calc.get_SMA()); - BlueTable state = new BlueTable(_schema); - state.addRow(nr); - DataPacket dp_out = BlueDataUtility.build_dp_inc(state, myLastValue as DataPacket); - myFireEvent(dp_out); - } - } - -} + [DataStreamUserRespresentation("SMA")] + public class SMAStream : ViewStream { + public SMAStream(ViewStream ohlc_str, int period) { + GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract", + (rows, idx) => { + IBlueRow ir = rows.ElementAt((int)idx); + IEnumerable closes = BlueDataUtility.get_colvalues(rows, "close", (int)idx + 1, period + 2); + double rsi = ChartingHelper.calculate_RSI(closes.ToList(), period); + IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "rsi")); + MutableBlueRow nr = new MutableBlueRow(schema, ir); + nr.setValue("rsi", rsi); + return nr; + }); + mySetRealSource(rs); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SecurityRenkoStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SecurityRenkoStream.cs deleted file mode 100644 index 9e4106729..000000000 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SecurityRenkoStream.cs +++ /dev/null @@ -1,49 +0,0 @@ -using System; - -namespace BCG { - /* - ---------------------------------------------------------------------- - *Expression* - ---------------------------------------------------------------------- - r=filter(bbo(),"sec='BTC-PERPETUAL'") - r=math(r,"wt_mid_price = (bid_price*bid_sz + ask_price*ask_sz)/(bid_sz+ask_sz)") - r=timeSeries(r) - r=math(r,"sz = (bid_sz + ask_sz) * 1.0") - r=filter(r,"sz > 0") - r=renko(r,"wt_mid_price", 2.0) - - ---------------------------------------------------------------------- - *Schema* - ---------------------------------------------------------------------- - id System.Int32 - entry_price System.Double - current_price System.Double - brick_low System.Double - brick_high System.Double - brick_transition System.String - n_updates_within System.Int32 - */ - - - public class SecurityRenkoStream : ViewStream { - public SecurityRenkoStream(int sec_key, double brick_sz) { - ViewStream r; - - r = new FilterStream(new BBOStream(), - $"sec_key={sec_key}"); - r = new MathStream(r, - "wt_mid_price = (bid_price*bid_sz + ask_price*ask_sz)/(bid_sz+ask_sz)"); - r = new TimeSeriesStream(r); - r = new MathStream(r, - "sz = (bid_sz + ask_sz) * 1.0"); - r = new FilterStream(r, - "sz > 0"); - r = new RenkoStream(r, - "wt_mid_price", - brick_sz); - - - mySetRealSource(r); - } - } -} diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/TradeBarsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/TradeBarsStream.cs deleted file mode 100644 index 5fa0cf634..000000000 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/TradeBarsStream.cs +++ /dev/null @@ -1,161 +0,0 @@ -using BCG; -using BCG.Utilities.General; - -using System; - -namespace bts { - [DataStreamUserRespresentation("tradebars")] - public class TradeBarsStream : ViewStream, IWorker { - - public class TradeBar { - static double[] calculate_log_returns(double[] prices) { - // Calculate logarithmic returns - double[] returns = new double[prices.Length - 1]; - for (int i = 1; i < prices.Length; i++) { - returns[i - 1] = Math.Log(prices[i] / prices[i - 1]); - } - return returns; - } - - static double calculate_volatility(double[] returns) { - // Calculate standard deviation of logarithmic returns - double mean = returns.Average(); - double sum_of_squares = returns.Sum(x => Math.Pow(x - mean, 2)); - double variance = sum_of_squares / (returns.Length - 1); - double volatility = Math.Sqrt(variance); - - return volatility; - } - public void add_trade(IBlueRow r) { - double price = r.get("price"); - //prices.Add(price); - double qty = r.get(col_name: "qty"); - if (n_entries == 0) { - open = price; - low = price < low ? price : low; - high = price > high ? price : high; - } - n_entries++; - volume += qty; - vwap_sum += price * qty; - close = price; - if ("Buy".Equals(r.get("aggressor"))) { - n_buys_aggs++; - } else { - n_sell_aggs++; - } - } - private List prices = new List(); - public readonly string security; - public double low = Double.MaxValue, high, open, close, median, std_dev, vwap_sum, volume; - public uint n_buys_aggs, n_sell_aggs, n_entries; - public DateTime bar_start = DateTime.Now; - public TradeBar(string security_) { - security = security_; - } - public double volatility { - get { - //if (prices.Count == 1) - return 0.0; - var log_returns = calculate_log_returns(prices.ToArray()); - return calculate_volatility(log_returns); - } - } - } - - private readonly int _bar_period_seconds; - private readonly ViewStreamDatabase _vsd; - private ViewStream _timesales_stream; - private ViewStream _bbo_stream; - private IDictionary _sec_bar_map = new Dictionary(); - private bool _dirty = false; - private TimeSpan _fire_period; - private DateTime _last_fire_time = DateTime.Now; - private BlueTable _table = new BlueTable(BlueDataUtility.form_schema("TradeBars", - Tuple.Create("id", TypeCode.Int32), - Tuple.Create("bar_start", TypeCode.DateTime), - Tuple.Create("bar_end", TypeCode.DateTime), - Tuple.Create("day_of_week", TypeCode.String), - Tuple.Create("n_trades", TypeCode.UInt32), - Tuple.Create("security", TypeCode.String), - Tuple.Create("sec_key", TypeCode.UInt32), - Tuple.Create("low", TypeCode.Double), - Tuple.Create("high", TypeCode.Double), - Tuple.Create("open", TypeCode.Double), - Tuple.Create("close", TypeCode.Double), - Tuple.Create("vwap_sum", TypeCode.Double), - Tuple.Create("volume", TypeCode.Double), - Tuple.Create("n_buy_aggs", TypeCode.UInt32), - Tuple.Create("n_sell_aggs", TypeCode.UInt32), - Tuple.Create("volatility", TypeCode.Double))); - - - public TradeBarsStream(int bar_period_seconds) - : this(bar_period_seconds, ViewStreamDatabase.instance) { } - public TradeBarsStream(int bar_period_seconds, ViewStreamDatabase vsd) { - _vsd = vsd; - _bar_period_seconds = bar_period_seconds; - _timesales_stream = new TimeSalesStream(10000, vsd); - _fire_period = TimeSpan.FromSeconds(bar_period_seconds); - _bbo_stream = new BBOStream(_vsd); - } - - protected override void myOnFirstSubscriber() { - _timesales_stream.onChange += _on_timesales; - _bbo_stream.onChange += _on_bbo; - _vsd.addWorker(this); - } - - protected override void myOnLastSubscriber() { - _vsd.removeWorker(this); - _timesales_stream.onChange -= _on_timesales; - _bbo_stream.onChange -= _on_bbo; - } - - public void doWork() { - if (DateTime.Now - _last_fire_time > _fire_period) { - if (_dirty) { - _dirty = false; - _last_fire_time = DateTime.Now; - BlueTable added_rows = new BlueTable(_table.schema); - foreach (var kv in _sec_bar_map) { - var sec_key = kv.Key; - var tbar = kv.Value; - IBlueRow new_row = new BlueRow(_table.schema, _table.Count, tbar.bar_start, DateTime.Now, - tbar.bar_start.DayOfWeek.ToString(), - tbar.n_entries, tbar.security, sec_key, tbar.low, tbar.high, tbar.open, - tbar.close, tbar.vwap_sum, tbar.volume, tbar.n_sell_aggs, tbar.n_sell_aggs, - tbar.volatility); - added_rows.addRow(new_row); - _table.addRow(new_row); - } - _sec_bar_map.Clear(); - DataPacket dp_out = new DataPacket(_table, added_rows, new BlueTable(_table.schema)); - myFireEvent(dp_out); - } - } - } - - private void _on_timesales(object obj) { - DataPacket dp = obj as DataPacket; - foreach (IBlueRow r in dp.addedRows) { - uint sec_key = r.get("sec_key"); - TradeBar tbar = null; - if (!_sec_bar_map.TryGetValue(sec_key, out tbar)) { - tbar = new TradeBar(r.get("sec")); - _sec_bar_map.Add(sec_key, tbar); - } - _dirty = true; - tbar.add_trade(r); - } - - - } - - private void _on_bbo(object obj) { - - } - } - - -} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs new file mode 100644 index 000000000..57392b55b --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs @@ -0,0 +1,56 @@ +namespace bts.SilverPine { + using bts; + using BCG; + using MongoDB.Driver.Linq; + using System.ComponentModel.DataAnnotations; + + [DataStreamUserRespresentation("VWAP")] + public class VWAPStream : ViewStream { + // returns the VWAP stream for each contract + private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) { + var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt); + ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params); + // stream is grouped per contract and then vwap is calculated per contract + GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract", + (rows, idx) => { + Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}"); + IBlueRow ir = rows.ElementAt((int)idx); + var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt); + var volumeprices = sel_rows.Select(r => Tuple.Create(r.get("volume"), r.get("close"))).ToList(); + double vwap = ChartingHelper.calculate_VWAP(volumeprices); + IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap")); + MutableBlueRow nr = new MutableBlueRow(schema, ir); + //vwap = -313.987; + nr.setValue("vwap", vwap); + return nr; + }); + return rs; + } + // this assumes the ohlc_str has diff contracts + public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) { + Tuple ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract); + string symbol = String.Empty; + symbol = ccat_sym_contr.Item2; + var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt); + ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt); + // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract, + // whihc we do with FutureActivesContractStream. If the arg was a ocntract we'll simply filter it + ViewStream rs = vwap_str; + switch (ccat_sym_contr.Item1) { + case ContractUtils.ContractCategory.FutureString: + case ContractUtils.ContractCategory.SpreadString: + rs = new FilterStream(vwap_str, r => r.get("contract").CompareTo(ccat_sym_contr.Item2) == 0); + break; + case ContractUtils.ContractCategory.Symbol: + rs = new FuturesActiveContractStream(vwap_str); + break; + } + rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0); + mySetRealSource(rs); + } + + public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt) + : + this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs new file mode 100644 index 000000000..14ddb97ae --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs @@ -0,0 +1,78 @@ +namespace bts.SilverPine { + using BCG; + + using bts; + // Given volume data per date for all futures (all exps + spreads) it selects the + // the future with the highest volume + [DataStreamUserRespresentation("FuturesActiveContract")] + public class FuturesActiveContractStream : ViewStream, INonIncGroupByEvaluator { + + public FuturesActiveContractStream(string future_name) + // the active contract's resoulution is a day, so we used + : this(new OHLCVStream(future_name, + // uses Historic Daily bars to find the acive contract + new OHLCVStream.OHLCVParams(OHLCVStream.Source.SQL, OHLCVStream.OHLCVType.HISTORIC, OHLCVStream.OHLCVBarSize.ONE_DAY, + OHLCVStream.get_default_historic_range()))) { } + + // simply finds the most active (highest volume) contract per day + public FuturesActiveContractStream(ViewStream ohlcv_stream) { + var rs = new FilterStream(ohlcv_stream, r => { + ContractUtils.ContractType t = ContractUtils.get_type(r.get("contract")); + return t == ContractUtils.ContractType.Future; + }); + mySetRealSource(new NonIncGroupByStream(rs, this)); + } + + public IBlueRowCollection evaluate(IDictionary> grouped_rows) { + Func is_future = (c) => ContractUtils.get_category(c) == ContractUtils.ContractCategory.FutureString; + BlueTable rv = null; + Func, IBlueRow> reduce = (rows) => { + var future_rows = rows.Where(r => is_future(r.get("contract"))); + IBlueRow rv = future_rows.ElementAt(0); + foreach (IBlueRow r in future_rows) { + if (r.get("volume") > rv.get("volume")) + rv = r; + } + return rv; + }; + foreach (var kv in grouped_rows) { + var rows = kv.Value; + if (rows == null || rows.Count == 0) + continue; + IBlueRow rr = reduce(rows); + rv = rv ?? new BlueTable(rr.schema); + rv.addRow(rr); + } + return rv; + } + public string[] get_group_by_cols() { + return new string[] { "ts_event" }; + } + } + + // Reduces the OHLCV stream by date and by either contract (if given explicty) or active contract + // is the first arg is a symbol and not a contract + public class FilteredOHLCVStream : ViewStream { + public FilteredOHLCVStream(string symbol_or_contract) + : + this(symbol_or_contract, DateTime.Today.Date.Subtract(TimeSpan.FromDays(1)), DateTime.MinValue) { } + + public FilteredOHLCVStream(string symbol_or_contract, DateTime dt1, DateTime dt2) { + Tuple tup = ContractUtils.get_symbol_category(symbol_or_contract); + var ohlcv_params = OHLCVStream.get_params(dt1, dt2); + ViewStream rs = new OHLCVStream(tup.Item2, ohlcv_params); + switch (tup.Item1) { + case ContractUtils.ContractCategory.FutureString: + case ContractUtils.ContractCategory.SpreadString: + rs = new FilterStream(rs, $"contract = '{symbol_or_contract}'"); + break; + case ContractUtils.ContractCategory.Symbol: + rs = new FuturesActiveContractStream(rs); + break; + } + rs = new FilterStream(rs, r => r.get_datetime("ts_event") <= dt1 && + r.get_datetime("ts_event") >= dt2); + mySetRealSource(rs); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesRollStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesRollStream.cs new file mode 100644 index 000000000..01783ee8a --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesRollStream.cs @@ -0,0 +1,16 @@ +namespace bts.SilverPine { + using bts; + using BCG; + + // it uses the volume - better to use open interst perhap - to detect the rollover, is when a new + // futuere exp becomes the highest volume one + [DataStreamUserRespresentation("FuturesRoll")] + public class FuturesRollStream : ViewStream { + public FuturesRollStream(string future_symbol) { + //ViewStream ohlcv_str = new OHLCVStream(future_symbol); + ViewStream rs = new FuturesActiveContractStream(future_symbol); + rs = new ColumnEvolutionStream(rs, "ts_event", "contract"); + mySetRealSource(rs); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVPriceDiffStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVPriceDiffStream.cs new file mode 100644 index 000000000..f5fc9dfd4 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVPriceDiffStream.cs @@ -0,0 +1,36 @@ +namespace bts.SilverPine { + using BCG; + + [DataStreamUserRespresentation("OHLCVPriceDiff")] + public class OHLCVPriceDiffStream : ViewStream { + // TODO, abstract this out + private static ViewStream _get_ohlcv_steam(string symbol_or_contract, DateTime d1, DateTime d2) { + var tup = ContractUtils.get_symbol_category(symbol_or_contract); + var ohlcv_params = OHLCVStream.get_params(d1, d2); + var cont_cat = tup.Item1; + string symbol = tup.Item2; + ViewStream rs = new OHLCVStream(symbol, ohlcv_params); + switch (cont_cat) { + case ContractUtils.ContractCategory.FutureString: + case ContractUtils.ContractCategory.SpreadString: + rs = new FilterStream(rs, $"contract = '{symbol_or_contract}'"); + break; + case ContractUtils.ContractCategory.Symbol: + rs = new FuturesActiveContractStream(rs); + break; + } + return rs; + } + + public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2) + : this(symbol_or_contract, d1, d2, "close") { } + + public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2, string col) { + ViewStream rs = _get_ohlcv_steam(symbol_or_contract, d1, d2); + rs = new ColumnDifferenceStream(rs, r => d1.Equals(r.get_datetime("ts_event")), + r => d2.Equals(r.get_datetime("ts_event")), col, col); + mySetRealSource(rs); + } + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs new file mode 100644 index 000000000..7291a773b --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs @@ -0,0 +1,117 @@ +namespace bts.SilverPine { + using bts; + using BCG; + using MongoDB.Driver; + [DataStreamUserRespresentation("OHLCV")] + // This stream retreives the OHLCV table as is, and fires it as-is. + // The only logic here is as follows: + // 1. deciding bw sql table or pubsub + // 2. deciding bw live, historic or append(live, historic) + // 3. deciding size of bar + // (the logic for2 and 3 is in get_params) + public class OHLCVStream : ViewStream { + public enum Source { + SQL, + PubSub + } + public enum OHLCVType { + LIVE, + HISTORIC, + LIVE_AND_HISTORIC + + }; + public enum OHLCVBarSize { + ONE_DAY = 0, + ONE_HOUR, + ONE_MIN, + ONE_SEC + }; + public class OHLCVParams { + public readonly Source source = Source.SQL; + public readonly OHLCVType type = OHLCVType.HISTORIC; + public readonly OHLCVBarSize bar_size; + public readonly Tuple optional_dt_range; + public OHLCVParams(Source source_, OHLCVType type_, OHLCVBarSize bar_size_, + Tuple optional_dt_range_) { + source = source_; + type = type_; + bar_size = bar_size_; + optional_dt_range = optional_dt_range_; + } + } + + public static OHLCVParams get_params(DateTime dt1, DateTime dt2) { + Func today = (dt) => dt.Date >= DateTime.Now.Date; + TimeSpan period = dt1 > dt2 ? dt1 - dt2 : dt2 - dt1; + bool has_today = today(dt1) || today(dt2); + bool has_history = !today(dt1) && !today(dt2); + Source source = Source.SQL; + OHLCVType type = (has_today && has_history) ? OHLCVType.LIVE_AND_HISTORIC : + ((has_today && !has_history) ? OHLCVType.LIVE : OHLCVType.HISTORIC); + // TODO, for less than a day, its defaulted to 1 min + OHLCVBarSize bar_size = period >= TimeSpan.FromDays(1.0) ? OHLCVBarSize.ONE_DAY : OHLCVBarSize.ONE_MIN; + Tuple optional_dt_range = new Tuple(dt1, dt2); + return new OHLCVParams(source, type, bar_size, optional_dt_range); + } + + public static Tuple get_default_historic_range() { + return Tuple.Create(DateTime.Now.Date.Subtract(TimeSpan.FromDays(1)), DateTime.MinValue); + } + + private static string get_table_from_barsize(OHLCVBarSize bar_size, string symbol) { + switch (bar_size) { + case OHLCVBarSize.ONE_DAY: + return $"{symbol}_OHLCV_1DAY"; + case OHLCVBarSize.ONE_SEC: + case OHLCVBarSize.ONE_MIN: + case OHLCVBarSize.ONE_HOUR: + return $"{symbol}_OHLCV_1MIN"; + } + Log.assert(false); + return String.Empty; + } + + private Source _sourse = Source.SQL; // get from config + private readonly string _location = "C:\\Users\\m_tah\\development\\data\\"; + private readonly string _db_name = "stream_data.db"; + // the symbol is used ot retreive the correct table, the data flows as-is from the table + // with some name / type changes + public OHLCVStream(string symbol, OHLCVParams ohlcv_params) { + switch (ohlcv_params.type) { + case OHLCVType.LIVE: + case OHLCVType.LIVE_AND_HISTORIC: + Log.assert(false); + break; + case OHLCVType.HISTORIC: + // TODO: assuming 1 day only for historic + string table_name = get_table_from_barsize(ohlcv_params.bar_size, symbol);// $"{symbol}_OHLCV_1DAY"; + //string table_name = $"{symbol}_{ohlcv_params.bar_size}"; + string db_str = $"Data Source={_location}{_db_name};Version=3"; + string sql_str = $"select * from {table_name}"; + // "Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;" + ViewStream rs = new SQLiteStream(db_str, sql_str); + rs = new SchemaGuesserStream(rs); + if (ohlcv_params.bar_size == OHLCVBarSize.ONE_DAY) + rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime); + rs = new NewProjectStream(rs, "symbol:contract"); + mySetRealSource(rs); + break; + } + } + public OHLCVStream(string symbol) + : + this(symbol, new OHLCVParams(Source.SQL, OHLCVType.HISTORIC, OHLCVBarSize.ONE_DAY, get_default_historic_range())) { } + + public OHLCVStream(string symbol, OHLCVBarSize bar_size) + : + this(symbol, new OHLCVParams(Source.SQL, OHLCVType.HISTORIC, bar_size, get_default_historic_range())) { + } + + public OHLCVStream(string symbol, string bar_size) + : + this(symbol, Enum.Parse(bar_size)) { } + + + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs new file mode 100644 index 000000000..6e231e758 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs @@ -0,0 +1,48 @@ +namespace bts.SilverPine { + using BCG; + + using bts; + + using DnsClient.Protocol; + + using MongoDB.Driver; + + [DataStreamUserRespresentation("Settlement")] + public class SettlementPriceStream : ViewStream { + + public SettlementPriceStream(string future_name, string date_txt) + : this(future_name, date_txt, String.Empty) { } + + // public SettlementPriceStream(string future_name, string begin_dt, string end_dt) + // : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { } + + public SettlementPriceStream(string symbol_or_contract, string begin_dt_txt, string end_dt_txt) { + DateTime begin_dt = DateTime.Parse(begin_dt_txt); + DateTime end_dt = String.IsNullOrEmpty(end_dt_txt) ? DateTime.MinValue : DateTime.Parse(end_dt_txt); + Tuple ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract); + ContractUtils.ContractCategory contr_cat = ccat_sym_contr.Item1; + string symbol = ccat_sym_contr.Item2; + ViewStream settl_str = new OHLCVStream(symbol); + // TODO: close != settlement, get settlement from somewhere + ViewStream rs = new NewProjectStream(settl_str, "close:settlement"); + switch (contr_cat) { + case ContractUtils.ContractCategory.FutureString: + case ContractUtils.ContractCategory.SpreadString: + rs = new FilterStream(settl_str, r => r.get("contract").CompareTo(symbol_or_contract) == 0); + break; + case ContractUtils.ContractCategory.Symbol: + rs = new FuturesActiveContractStream(settl_str); + break; + } + if (DateTime.MinValue.Equals(end_dt)) { + rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0); + } + rs = new FilterStream(rs, r => { + DateTime dt = r.get_datetime("ts_event"); + return dt <= begin_dt && dt >= end_dt; + }); + mySetRealSource(rs); + } + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/BarsAggregatingStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/BarsAggregatingStream.cs new file mode 100644 index 000000000..86caa300f --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/BarsAggregatingStream.cs @@ -0,0 +1,32 @@ +namespace bts.SilverPine { + using BCG; + // Takes multiple bars and collapses them into one, which is aggregated over a bigger time period. + // All bars belonging to a common group_by_col, are aggregated + public class BarsAggregatorStream : ViewStream { + public BarsAggregatorStream(ViewStream ohlcv_stream, params string[] group_by_cols) { + ViewStream rs = new NonIncGroupByStream(ohlcv_stream, + () => { return group_by_cols; }, + (group_rows) => { + var sorted_rows = group_rows.OrderBy(r => r.get_datetime("ts_event")); + IBlueRow first_row = sorted_rows.First(), + last_row = sorted_rows.Last(); + double open = first_row.get("open"); + double close = last_row.get("close"); + double low = sorted_rows.Min(r => r.get("low")); + double high = sorted_rows.Max(r => r.get("high")); + double volume = 0.0; + foreach (IBlueRow r in sorted_rows) + volume += r.get("volume"); + MutableBlueRow rv = new MutableBlueRow(first_row); + rv.setValue("open", open); + rv.setValue("close", close); + rv.setValue("low", low); + rv.setValue("high", high); + rv.setValue("volume", volume); + return new IBlueRow[] { rv }.ToList(); + } + ); + mySetRealSource(rs); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/BarsSearcherStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/BarsSearcherStream.cs new file mode 100644 index 000000000..5626f7114 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/BarsSearcherStream.cs @@ -0,0 +1,32 @@ +namespace bts.SilverPine { + using BCG; + + // this is an internal stream. its job is to take an OHLCV stream + // and filter it for + // 1. symbol (table_name and active contract) and + // 2. time (can be a day of the week, month, and a time of the day period) + public class BarsSearcherStream : ViewStream { + public BarsSearcherStream(string symbol, OHLCVStream.OHLCVBarSize bar_size, + string day_of_week, TimeSpan tod_begin, TimeSpan tod_end) { + ViewStream rs = new OHLCVStream(symbol, bar_size); + rs = new DateTimeExpanstionStream(rs, "ts_event"); + rs = new FilterStream(rs, r => { + DateTime ts = r.get_datetime("ts_event"); + return r.get("ts_event_day") == day_of_week.ToString() && + ts.TimeOfDay >= tod_begin && ts.TimeOfDay <= tod_end; + }); + // look at symbol and either filter ir ot do the active contracts + // assuming its symbol + rs = new NonIncJoinStream(rs, "ts_event_date", "open high low close volume contract ts_event", + new FuturesActiveContractStream(symbol), "ts_event", "ts_event:date contract:active_contract"); + // filter it for the active contract only + rs = new FilterStream(rs, r => String.Compare(r.get("contract"), r.get("active_contract")) == 0); + mySetRealSource(rs); + } + public BarsSearcherStream(string symbol, string bar_size, + string day_of_week, string tod_begin, string tod_end) + : + this(symbol, Enum.Parse(bar_size), day_of_week, + TimeSpan.Parse(tod_begin), TimeSpan.Parse(tod_end)) { } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs deleted file mode 100644 index 14cea626b..000000000 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/TimeSeriesGroupByStream.cs +++ /dev/null @@ -1,130 +0,0 @@ -namespace BCG { - using bts.api.orderman; - - enum TSGroupByOperation { - OHLC = 0, - Statistics - } - - [DataStreamUserRespresentation("timeseriesanalysis")] - public class TimeSeriesGroupByStream : ViewStream { - public TimeSeriesGroupByStream(ViewStream md_str, string timebucket_col, string output_col, string operation) { - ViewStream rs = new NonIncGroupByStream(md_str, new OHLCNonIncGroupByEvaluator(timebucket_col, output_col)); - mySetRealSource(rs); - } - } - public class OHLC { - public OHLC(double open_args, double high_args, double low_args, double close_args) { - open = open_args; - high = high_args; - low = low_args; - close = close_args; - } - public readonly double open, high, low, close; - } - public class OHLCNonIncGroupByEvaluator : INonIncGroupByEvaluator { - private string _timestamp_col; - private string _output_col; - private IBlueTableSchema _out_schema; - public OHLCNonIncGroupByEvaluator(string timestamp_col, string output_col) { - _timestamp_col = timestamp_col; - _output_col = output_col; - } - - string open_col { - get { - return $"{_output_col}_open"; ; - } - } - string high_col { - get { - return $"{_output_col}_high"; ; - } - } - string low_col { - get { - return $"{_output_col}_low"; ; - } - } - string close_col { - get { - return $"{_output_col}_close"; ; - } - } - private void _build_schema(IBlueTableSchema input_schema) { - List> new_cols = new List>(); - ColumnInfo ci = input_schema[_output_col]; - new_cols.Add(Tuple.Create(ci.colType, open_col)); - new_cols.Add(Tuple.Create(ci.colType, high_col)); - new_cols.Add(Tuple.Create(ci.colType, low_col)); - new_cols.Add(Tuple.Create(ci.colType, close_col)); - // should hide the output col - _out_schema = BlueDataUtility.add_columns(input_schema, new_cols.ToArray()); - } - - public IBlueRowCollection evaluate(IDictionary> grouped_rows) { - if (_out_schema == null) { - if (grouped_rows.Count > 0) { - foreach (var kv in grouped_rows) { - var rows = kv.Value; - if (rows != null && rows.Count > 0) { - _build_schema(rows.ElementAt(0).schema); - } - } - } - } - BlueTable rv = new BlueTable(_out_schema); - foreach (var kv in grouped_rows) { - int id = kv.Key; - IEnumerable rows = kv.Value; - // sort rows on time to compute o,c correctly?? - if (rows != null && rows.Count() > 0) { - IBlueRow reprow = rows.ElementAt(0); - IEnumerable vals = BlueDataUtility.get_colvalues(rows, _output_col); - OHLC ohlc = compute_ohlc(vals); - MutableBlueRow nr = new MutableBlueRow(_out_schema, reprow); - nr.setValue("id", kv.Key); - nr.setValue(open_col, ohlc.open); - nr.setValue(high_col, ohlc.high); - nr.setValue(low_col, ohlc.low); - nr.setValue(close_col, ohlc.close); - rv.addRow(nr); - } - } - return rv; - } - - public string[] get_group_by_cols() { - return new string[] { _timestamp_col }; - } - - static OHLC compute_ohlc(IEnumerable prices) { - double o = prices.ElementAt(0); - double c = prices.ElementAt(prices.Count() - 1); - double l = prices.Min(); - double h = prices.Max(); - return new(o, h, l, c); - } - } - [DataStreamUserRespresentation("RSI")] - public class RSIStream : ViewStream { - // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for - public RSIStream(ViewStream md_str, string timebucket_column, string output_col) { - - } - } - [DataStreamUserRespresentation("Bollinger")] - public class BollingerBandStream : ViewStream { - // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for - public BollingerBandStream(ViewStream md_str, string timebucket_column, string output_col) { - } - } - [DataStreamUserRespresentation("OHLC")] - public class OHLCStream : ViewStream { - // for ms_str expects a timeseries with a timestamp col and an output col (price col) to calculate the ohlc for - public OHLCStream(ViewStream md_str, string timebucket_column, string output_col) { - ViewStream rs = new TimeSeriesGroupByStream(md_str, timebucket_column, output_col, TSGroupByOperation.OHLC.ToString()); - mySetRealSource(rs); - } - } -} \ No newline at end of file diff --git a/src/lib/config/profiles/default/toggles.conf b/src/lib/config/profiles/default/toggles.conf index b5ebf2442..2200ca1ee 100644 --- a/src/lib/config/profiles/default/toggles.conf +++ b/src/lib/config/profiles/default/toggles.conf @@ -114,7 +114,12 @@ VolSurfaceUsage { items: { option_name: "SPX" option_venue: VENUE_CBOE - vs_name: "SPX_moneyness" + vs_name: "Main" + } + items: { + option_name: "SPXW" + option_venue: VENUE_CBOE + vs_name: "Main" } }