From a7189628b1823909cb1b36368c06daed5a2ffc3a Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Mon, 18 Dec 2023 12:42:14 -0500 Subject: [PATCH] Adding EMA, SMA and RenkoStreams --- .../Primitives/AddColumns/AddColumnsStream.cs | 24 +- .../Primitives/FileStreams/PlaybackStream.cs | 348 +++++++++--------- .../Primitives/Misc/TimeSeriesStream.cs | 96 +++-- .../BlueData/CircularRowCollection.cs | 34 +- .../DynamicLoading/PrimitiveManager.cs | 2 +- .../DataStreamQueryAnalyzer/MainForm.cs | 2 +- .../SharedStreams/Business/ChartsUtils.cs | 312 ++++++++++++++++ .../Business/MarketData/Charting/EMAStream.cs | 54 +++ .../MarketData/Charting/RenkoStream.cs | 87 +++++ .../Business/MarketData/Charting/SMAStream.cs | 54 +++ .../Business/MarketData/TimeSalesStream.cs | 59 ++- .../cmake/GetWindowsClientLinkLibraries.cmake | 2 +- 12 files changed, 782 insertions(+), 292 deletions(-) create mode 100644 src/DataStreamSuite/SharedStreams/Business/ChartsUtils.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/EMAStream.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RenkoStream.cs create mode 100644 src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SMAStream.cs diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs index c63654497..bb2500791 100644 --- a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs @@ -1,13 +1,13 @@ -using System; -using System.Data; -using System.Collections; -using BCG.Utilities; -using BCG.Utilities.General; -namespace BCG -{ +// using System; +// using System.Data; +// using System.Collections; +// using BCG.Utilities; +// using BCG.Utilities.General; +// namespace BCG +// { - public class AddColumnsStream : MapStream - { - public AddColumnsStream(ViewStream vs, string columnDef) : base (vs, new ColumnAdder (columnDef)) {} - } -} \ No newline at end of file +// public class AddColumnsStream : MapStream +// { +// public AddColumnsStream(ViewStream vs, string columnDef) : base (vs, new ColumnAdder (columnDef)) {} +// } +// } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs b/src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs index 8f6fad9a3..6e2674347 100644 --- a/src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs @@ -1,190 +1,190 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.IO; -using System.Runtime.Serialization.Formatters.Binary; -using BCG.Utilities.General; -using bts.collections; -using System.Reflection.Metadata; +// using System; +// using System.Collections.Generic; +// using System.Linq; +// using System.Text; +// using System.IO; +// using System.Runtime.Serialization.Formatters.Binary; +// using BCG.Utilities.General; +// using bts.collections; +// using System.Reflection.Metadata; -namespace BCG { +// namespace BCG { - public class PlaybackStream : ViewStream, IWorker { - private readonly PlaybackType _type; - private BlueTable _current_table; - private IClientDeserializer _deserilializer = new ClientUniversalDeserializer(); - // these are the packets already downloaded from the current file - private List _packets = new List(); - //private IEnumerable _file_streams = new List(); - private IEnumerable _file_names; - private Tuple _current_fs = new Tuple(-1, null); - private DateTime _playback_t0, _playback_t1; - private bool _all_files_read = false; - SnapshotData _snapshot_data = null; // only initialized when in snapshot mode +// public class PlaybackStream : ViewStream, IWorker { +// private readonly PlaybackType _type; +// private BlueTable _current_table; +// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer(); +// // these are the packets already downloaded from the current file +// private List _packets = new List(); +// //private IEnumerable _file_streams = new List(); +// private IEnumerable _file_names; +// private Tuple _current_fs = new Tuple(-1, null); +// private DateTime _playback_t0, _playback_t1; +// private bool _all_files_read = false; +// SnapshotData _snapshot_data = null; // only initialized when in snapshot mode - public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) { - _type = PlaybackType.Streaming; - _playback_t0 = PersistorStream.parse_datetime(datetime_start); - _playback_t1 = PersistorStream.parse_datetime(datetime_end); - string token_directory_name = $"{directory_name}\\{token}"; - bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}"); - List files_in_range = new List(); - if (Directory.Exists(token_directory_name)) { - _file_names = - Directory.GetFiles(token_directory_name).Where(f => { - DateTime file_time = PersistorStream.parse_datetime_from_filename(f); - return file_time > _playback_t0 && file_time < _playback_t1; - }); - } - } +// public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) { +// _type = PlaybackType.Streaming; +// _playback_t0 = PersistorStream.parse_datetime(datetime_start); +// _playback_t1 = PersistorStream.parse_datetime(datetime_end); +// string token_directory_name = $"{directory_name}\\{token}"; +// bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}"); +// List files_in_range = new List(); +// if (Directory.Exists(token_directory_name)) { +// _file_names = +// Directory.GetFiles(token_directory_name).Where(f => { +// DateTime file_time = PersistorStream.parse_datetime_from_filename(f); +// return file_time > _playback_t0 && file_time < _playback_t1; +// }); +// } +// } - protected override void myOnFirstSubscriber() { - FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read); - _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs); - PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time); - if (snapshot_packet != null) { - foreach (PlaybackPacket p in _snapshot_data.packets) { - byte[] ops; - Array row_array; - _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array); - for (int i = 0; i < ops.Length; i++) { - IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; - byte op = ops[i]; - if (op == StreamOpCode.create) { - IBlueRowCollection full_table = rows_args[0]; - _current_table = new BlueTable(full_table); - } else if (op == StreamOpCode.multipleRemoveAdd) { - //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; - IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; - IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; - BlueDataUtility.absorbChanges(_current_table, dels, adds); - } else { - bts.Log.assert(false, $"Oprator: {op} is not expected"); - } - if (p.idx == snapshot_packet.idx) { - myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema))); - break; - } - } - } - } - } +// protected override void myOnFirstSubscriber() { +// FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read); +// _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs); +// PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time); +// if (snapshot_packet != null) { +// foreach (PlaybackPacket p in _snapshot_data.packets) { +// byte[] ops; +// Array row_array; +// _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array); +// for (int i = 0; i < ops.Length; i++) { +// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; +// byte op = ops[i]; +// if (op == StreamOpCode.create) { +// IBlueRowCollection full_table = rows_args[0]; +// _current_table = new BlueTable(full_table); +// } else if (op == StreamOpCode.multipleRemoveAdd) { +// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; +// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; +// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; +// BlueDataUtility.absorbChanges(_current_table, dels, adds); +// } else { +// bts.Log.assert(false, $"Oprator: {op} is not expected"); +// } +// if (p.idx == snapshot_packet.idx) { +// myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema))); +// break; +// } +// } +// } +// } +// } - private void _load_packets_from_current_file() { - int curr_idx = _current_fs.Item1; - FileStream curr_fs = _current_fs.Item2; - _packets = new List(); - _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs)); - } +// private void _load_packets_from_current_file() { +// int curr_idx = _current_fs.Item1; +// FileStream curr_fs = _current_fs.Item2; +// _packets = new List(); +// _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs)); +// } - private bool current_file_all_replayed { - get { - bool rv = true; - _packets.ForEach(p => rv = rv && p.replayed); - return rv; - } - } +// private bool current_file_all_replayed { +// get { +// bool rv = true; +// _packets.ForEach(p => rv = rv && p.replayed); +// return rv; +// } +// } - protected override void myOnLastSubscriber() { - var fs = _current_fs.Item2; - if (fs != null) { - fs.Close(); - } - ViewStreamDatabase.instance.removeWorker(this); - } +// protected override void myOnLastSubscriber() { +// var fs = _current_fs.Item2; +// if (fs != null) { +// fs.Close(); +// } +// ViewStreamDatabase.instance.removeWorker(this); +// } - // public event PlaybackStreamProgress onProgress; - // public event Action onFileExhausted; - private PlaybackPacket _locate_packet_to_replay() { - foreach (var p in _packets) { - // returns the first non replayed packet - if (!p.replayed) - return p; - } - else - { - double secondsElapsed = new BinaryReader(myFileReader).ReadDouble(); - myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed)); - if (onProgress != null) - onProgress(myFileReader.Position, myFileReader.Length); - } - } +// // public event PlaybackStreamProgress onProgress; +// // public event Action onFileExhausted; +// private PlaybackPacket _locate_packet_to_replay() { +// foreach (var p in _packets) { +// // returns the first non replayed packet +// if (!p.replayed) +// return p; +// } +// //else +// { +// double secondsElapsed = new BinaryReader(myFileReader).ReadDouble(); +// myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed)); +// if (onProgress != null) +// onProgress(myFileReader.Position, myFileReader.Length); +// } +// } - #region IWorker Members +// #region IWorker Members - public void doWork() { - if (_all_files_read) - return; - if (current_file_all_replayed) { - _load_next_file(); - } - PlaybackPacket curr_packet = _locate_packet_to_replay(); - if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway - byte[] ops; - Array row_array; - // what a stupid interface! - _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array); - _fire_packet(ops, row_array); - curr_packet.replay_timestamp = PersistorStream.epoch_now(); - } else { - PlaybackPacket prev_packet = _get_previous_packet(curr_packet); - // this is the time we need to fire (replay) this packet - long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous; - long now_epoch = PersistorStream.epoch_now(); - // fire when we are past the - if (now_epoch >= time_to_fire) { - byte[] ops; - Array row_array; - _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array); - _fire_packet(ops, row_array); - curr_packet.replay_timestamp = PersistorStream.epoch_now(); - } - } - } +// public void doWork() { +// if (_all_files_read) +// return; +// if (current_file_all_replayed) { +// _load_next_file(); +// } +// PlaybackPacket curr_packet = _locate_packet_to_replay(); +// if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway +// byte[] ops; +// Array row_array; +// // what a stupid interface! +// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array); +// _fire_packet(ops, row_array); +// curr_packet.replay_timestamp = PersistorStream.epoch_now(); +// } else { +// PlaybackPacket prev_packet = _get_previous_packet(curr_packet); +// // this is the time we need to fire (replay) this packet +// long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous; +// long now_epoch = PersistorStream.epoch_now(); +// // fire when we are past the +// if (now_epoch >= time_to_fire) { +// byte[] ops; +// Array row_array; +// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array); +// _fire_packet(ops, row_array); +// curr_packet.replay_timestamp = PersistorStream.epoch_now(); +// } +// } +// } - private void _fire_packet(byte[] ops, Array row_array) { - for (int i = 0; i < ops.Length; i++) { - IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; - byte op = ops[i]; - if (op == StreamOpCode.create) { - IBlueRowCollection full_table = rows_args[0]; - _current_table = new BlueTable(full_table); - //_current_table.addRows(full_table); - myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) : - (myLastValue as DataPacket).newRows)); - } else if (op == StreamOpCode.multipleRemoveAdd) { - //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; - IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; - IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; - BlueDataUtility.absorbChanges(_current_table, dels, adds); - myFireEvent(new DataPacket(_current_table, adds, dels)); - } else { - bts.Log.assert(false, $"Oprator: {op} is not expected"); - } +// private void _fire_packet(byte[] ops, Array row_array) { +// for (int i = 0; i < ops.Length; i++) { +// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; +// byte op = ops[i]; +// if (op == StreamOpCode.create) { +// IBlueRowCollection full_table = rows_args[0]; +// _current_table = new BlueTable(full_table); +// //_current_table.addRows(full_table); +// myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) : +// (myLastValue as DataPacket).newRows)); +// } else if (op == StreamOpCode.multipleRemoveAdd) { +// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; +// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; +// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; +// BlueDataUtility.absorbChanges(_current_table, dels, adds); +// myFireEvent(new DataPacket(_current_table, adds, dels)); +// } else { +// bts.Log.assert(false, $"Oprator: {op} is not expected"); +// } - } - } +// } +// } - private void _load_next_file() { - var new_idx = _current_fs.Item1 + 1; - if (new_idx >= _file_names.Count()) { - _all_files_read = true; - bts.Log.Gen.info("All files read"); - return; - } - var prev_fs = _current_fs.Item2; - if (prev_fs != null) - prev_fs.Close(); - var new_filename = _file_names.ElementAt(new_idx); - bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}"); - var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read); - _current_fs = new Tuple(new_idx, new_fs); - _load_packets_from_current_file(); - } +// private void _load_next_file() { +// var new_idx = _current_fs.Item1 + 1; +// if (new_idx >= _file_names.Count()) { +// _all_files_read = true; +// bts.Log.Gen.info("All files read"); +// return; +// } +// var prev_fs = _current_fs.Item2; +// if (prev_fs != null) +// prev_fs.Close(); +// var new_filename = _file_names.ElementAt(new_idx); +// bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}"); +// var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read); +// _current_fs = new Tuple(new_idx, new_fs); +// _load_packets_from_current_file(); +// } - #endregion - } -} +// #endregion +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs index 88ba32d32..3266b7a07 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs @@ -3,45 +3,37 @@ using Google.Protobuf.WellKnownTypes; using System; using BCG.Utilities.General; -namespace BCG -{ +namespace BCG { // Keeps adding changing rows to the stream, hence preserving the history // and creating a time series, for example this can be used to make bbo updates // into a time series of prices/qty - public class TimeSeriesStream : ViewStream - { + public class TimeSeriesStream : ViewStream { private BlueTable _table; private ViewStream _stream; private int _id = 0; - public TimeSeriesStream(ViewStream stream) - { + public TimeSeriesStream(ViewStream stream) { _stream = stream; } - protected override void myOnFirstSubscriber() - { + protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_onstream_change); } - protected override void myOnLastSubscriber() - { + protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_onstream_change); } - private void _onstream_change(object o) - { + private void _onstream_change(object o) { DataPacket dp = o as DataPacket; DateTime now = DateTime.Now; - if (_table == null) - { + if (_table == null) { IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName); schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp")); _table = new BlueTable(schema); } BlueTable added_rows = new BlueTable(_table.schema); - foreach (IBlueRow r in dp.addedRows) - { + foreach (IBlueRow r in dp.addedRows) { MutableBlueRow ar = new MutableBlueRow(_table.schema, r); ar.setValue("timestamp", now); ar.setValue("id", _id++); @@ -53,49 +45,49 @@ namespace BCG } } - public class TimeBucketStream : ViewStream, IStreamMapper - { - private ViewStream _src; - private readonly string _time_stamp_col; - private IBlueTableSchema _schema; - private int _bucket_size_seconds; + // public class TimeBucketStream : ViewStream, IStreamMapper + // { + // private ViewStream _src; + // private readonly string _time_stamp_col; + // private IBlueTableSchema _schema; + // private int _bucket_size_seconds; - public TimeBucketStream(ViewStream input, string time_stamp_col, int bucket_size_seconds) - { - _time_stamp_col = time_stamp_col; - _bucket_size_seconds = bucket_size_seconds; - var vs = new MapStream(input, this); - mySetRealSource(vs); - } + // public TimeBucketStream(ViewStream input, string time_stamp_col, int bucket_size_seconds) + // { + // _time_stamp_col = time_stamp_col; + // _bucket_size_seconds = bucket_size_seconds; + // var vs = new MapStream(input, this); + // mySetRealSource(vs); + // } - #region IStreamMapper Members - public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) - { - IBlueTableSchema rv = input_schema; - //rv = BlueDataUtility.remove_columns(rv, _time_stamp_col); - rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int)); - return rv; + // #region IStreamMapper Members + // public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) + // { + // IBlueTableSchema rv = input_schema; + // //rv = BlueDataUtility.remove_columns(rv, _time_stamp_col); + // rv = BlueDataUtility.add_column(rv, "bucket_number", typeof(int)); + // return rv; - } + // } - #endregion + // #endregion - #region IDynamicStreamMapper Members + // #region IDynamicStreamMapper Members - public IBlueRow map(IBlueRow row) - { - DateTime ts = row.get(_time_stamp_col); - TimeSpan time_of_day = (ts - ts.Date); - int seconds = Convert.ToInt32(time_of_day.TotalSeconds); - double buckets = seconds / _bucket_size_seconds; - int bucket_number = Convert.ToInt32(Math.Floor(buckets)); - MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row); - rv.setValue("bucket_number", bucket_number); - return rv; - } + // public IBlueRow map(IBlueRow row) + // { + // DateTime ts = row.get(_time_stamp_col); + // TimeSpan time_of_day = (ts - ts.Date); + // int seconds = Convert.ToInt32(time_of_day.TotalSeconds); + // double buckets = seconds / _bucket_size_seconds; + // int bucket_number = Convert.ToInt32(Math.Floor(buckets)); + // MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row); + // rv.setValue("bucket_number", bucket_number); + // return rv; + // } - #endregion - } + // #endregion + // } // public class ThrottleStream : ViewStream, IWorker // { diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/CircularRowCollection.cs b/src/DataStreamSuite/DataStreamCore/BlueData/CircularRowCollection.cs index b4a8d7607..b8db9429b 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/CircularRowCollection.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/CircularRowCollection.cs @@ -11,17 +11,17 @@ namespace BCG public class CircularRowCollection : OrderedBlueTable { private int myBufferSize; - public CircularRowCollection (IBlueTableSchema schema, int bufferSize) - : base (schema) - { - myBufferSize = bufferSize ; + public CircularRowCollection(IBlueTableSchema schema, int bufferSize) + : base(schema) + { + myBufferSize = bufferSize; } - public CircularRowCollection (IBlueRowCollection rows, int bufferSize) - : this (rows.schema, bufferSize) + public CircularRowCollection(IBlueTableSchema schema, IEnumerable rows, int bufferSize) + : this(schema, bufferSize) { foreach (IBlueRow r in rows) - addRow (r); + addRow(r); } public int bufferSize @@ -39,30 +39,30 @@ namespace BCG /// /// /// - public IBlueRow addRowWithReturn (IBlueRow r) + public IBlueRow addRowWithReturn(IBlueRow r) { - base.addRow (r); + base.addRow(r); if (myOrderedRows.Count > myBufferSize) { - IBlueRow retVal = myOrderedRows [0] as IBlueRow; - deleteRow (retVal); + IBlueRow retVal = myOrderedRows[0] as IBlueRow; + deleteRow(retVal); return retVal; } return null; } - public override void addRow (IBlueRow row) + public override void addRow(IBlueRow row) { - base.addRow (row); + base.addRow(row); if (myOrderedRows.Count > myBufferSize) - deleteRow (myOrderedRows [0] as IBlueRow); + deleteRow(myOrderedRows[0] as IBlueRow); } - public override void addRow (DataRow row) + public override void addRow(DataRow row) { - base.addRow (row); + base.addRow(row); if (myOrderedRows.Count > myBufferSize) - deleteRow (myOrderedRows [0] as IBlueRow); + deleteRow(myOrderedRows[0] as IBlueRow); } public void add_rows(IEnumerable rows_to_add, out IBlueRowCollection added_rows, out IBlueRowCollection deleted_rows) diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs index 14a74816a..0209fdd2b 100644 --- a/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs @@ -74,7 +74,7 @@ namespace BCG { break; } if (isVisible) { - Log.assert(myTypeLookupTable[userRep] == null); + Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName); myTypeLookupTable[userRep] = type; } } diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs index 4335054a0..2630394f1 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs @@ -382,7 +382,7 @@ namespace BCG { private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) { myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length; myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length; - myQueryTextBox.SelectionColor = Color.Black; + myQueryTextBox.SelectionColor = Color.White; myQueryTextBox.SelectionLength = 0; myTryExpression(myQueryTextBox.Text); } diff --git a/src/DataStreamSuite/SharedStreams/Business/ChartsUtils.cs b/src/DataStreamSuite/SharedStreams/Business/ChartsUtils.cs new file mode 100644 index 000000000..904e4529a --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/ChartsUtils.cs @@ -0,0 +1,312 @@ +using bts.config; + +using System; +using System.Collections; +using System.Collections.Generic; + +namespace BCG { + + public class RenkoBrick { + public enum Entry { + UP, DOWN, NEW + } + public int id; + public double entry_price; + public double current_price; + public double low; + public double high; + public int n_updates = 0; + public Entry entry = Entry.NEW; + public bool consistent => entry_price >= low && entry_price <= high; + } + public class RenkoCalculator { + public readonly double brick_size; + public RenkoBrick current_brick = null; + private bool _add_price(double price) { + if (current_brick == null) { + current_brick = new RenkoBrick { + id = 0, + entry_price = price, + current_price = price, + // center it around the current price + low = price - this.brick_size / 2, + high = price + this.brick_size / 2, + n_updates = 0, + entry = RenkoBrick.Entry.NEW + }; + return true; + } else { + if (price > current_brick.high) { + // new high brick + var old_brick = current_brick; + current_brick = new RenkoBrick { + id = old_brick.id + 1, + entry_price = price, + current_price = price, + low = old_brick.high, + high = old_brick.high + this.brick_size, + entry = RenkoBrick.Entry.UP + }; + return true; + } else if (price < current_brick.low) { + // new low brick + var old_brick = current_brick; + current_brick = new RenkoBrick { + id = old_brick.id + 1, + entry_price = price, + current_price = price, + high = old_brick.low, + low = old_brick.high - this.brick_size, + entry = RenkoBrick.Entry.DOWN + }; + return true; + } else { + current_brick.n_updates++; + current_brick.current_price = price; + return false; + } + } + } + + public IEnumerable update(double price) { + var rv = new List(); + do { + if (_add_price(price)) { + rv.Add(current_brick); + } + } + while (!current_brick.consistent); + return rv; + } + + public RenkoCalculator(double brick_size) { + this.brick_size = brick_size; + } + + } + + + public class EMACalculator { + private readonly double _smoothing_factor; + private double _last_EMA; + + public EMACalculator(double smoothing_factor) { + if (smoothing_factor <= 0 || smoothing_factor >= 1) { + throw new ArgumentException("Smoothing factor must be in the range (0, 1)."); + } + + this._smoothing_factor = smoothing_factor; + } + + public void add(double data) { + if (_last_EMA == 0) { + _last_EMA = data; + } else { + _last_EMA = _smoothing_factor * data + (1 - _smoothing_factor) * _last_EMA; + } + } + + public double get_EMA() { + return _last_EMA; + } + public static List calculate_EMA(List prices, double smoothing_factor) { + List emaValues = new List(); + + if (prices.Count == 0) { + return emaValues; + } + + emaValues.Add(prices[0]); // The first EMA value is the same as the first price + + for (int i = 1; i < prices.Count; i++) { + // EMA formula: EMA(t) = (1 - smoothingFactor) * EMA(t-1) + smoothingFactor * Price(t) + double ema = (1 - smoothing_factor) * emaValues[i - 1] + smoothing_factor * prices[i]; + emaValues.Add(ema); + } + + return emaValues; + } + } + + + static class Chartutils { + // Simple Moving Average + + public class SMACalculator { + private readonly int _window_size; + private Queue _data_queue; + private double _sum; + + public SMACalculator(int window_size) { + if (window_size <= 0) { + throw new ArgumentException("Window size must be greater than 0."); + } + + this._window_size = window_size; + this._data_queue = new Queue(window_size); + this._sum = 0; + } + + public void add(double data) { + if (_data_queue.Count == _window_size) { + double removed_value = _data_queue.Dequeue(); + _sum -= removed_value; + } + + _data_queue.Enqueue(data); + _sum += data; + } + + public double get_SMA() { + if (_data_queue.Count == 0) { + throw new InvalidOperationException("No data available to calculate SMA."); + } + + return _sum / _data_queue.Count; + } + // stateless functions that computed the SMA for a list of values + public static List calculate_SMA(List prices, int window_size) { + List sma_values = new List(); + for (int i = 0; i < prices.Count; i++) { + double sum = 0; + + // Calculate the sum of prices in the window + for (int j = Math.Max(0, i - window_size + 1); j <= i; j++) { + sum += prices[j]; + } + + // Calculate the SMA and add it to the result + double sma = sum / Math.Min(i + 1, window_size); + sma_values.Add(sma); + } + + return sma_values; + } + } + // Exponential Moving Average + + // Weighted Moving Average (WMA) + public static List calculate_WMA(List prices, List weights) { + List wmaValues = new List(); + + if (prices.Count != weights.Count || prices.Count == 0) { + return wmaValues; + } + + for (int i = 0; i < prices.Count; i++) { + double wma = 0; + + // Calculate WMA: WMA(t) = (P(t) * W(t) + P(t-1) * W(t-1) + ... + P(t-n) * W(t-n)) / (W(t) + W(t-1) + ... + W(t-n)) + double weightedSum = 0; + double weightSum = 0; + + for (int j = 0; j <= i; j++) { + weightedSum += prices[i - j] * weights[j]; + weightSum += weights[j]; + } + + if (weightSum != 0) { + wma = weightedSum / weightSum; + } + + wmaValues.Add(wma); + } + + return wmaValues; + } + //Adaptive Moving Average (AMA) + public static List calculate_AMA(List prices, int shortPeriod, int longPeriod, double multiplier) { + List amaValues = new List(); + + if (prices.Count == 0 || shortPeriod >= longPeriod) { + return amaValues; + } + + double[] erValues = new double[prices.Count]; + double[] scValues = new double[prices.Count]; + double[] fastSCValues = new double[prices.Count]; + double[] slowSCValues = new double[prices.Count]; + + erValues[0] = prices[0]; + scValues[0] = prices[0]; + fastSCValues[0] = prices[0]; + slowSCValues[0] = prices[0]; + + for (int i = 1; i < prices.Count; i++) { + double volatility = 0.1; // Initial volatility value + + // Efficiency Ratio (ER) calculation + double change = Math.Abs(prices[i] - prices[i - 1]); + double volatilitySum = 0; + + for (int j = i - shortPeriod + 1; j <= i; j++) { + volatilitySum += Math.Abs(prices[j] - prices[j - 1]); + } + + if (volatilitySum != 0) { + volatility = change / volatilitySum; + } + + erValues[i] = volatility == 0 ? 0 : Math.Pow(volatility, 2); + + // Smoothed ER (SC) calculation + fastSCValues[i] = (2.0 / (shortPeriod + 1)) * erValues[i] + (1.0 - 2.0 / (shortPeriod + 1)) * fastSCValues[i - 1]; + slowSCValues[i] = (2.0 / (longPeriod + 1)) * erValues[i] + (1.0 - 2.0 / (longPeriod + 1)) * slowSCValues[i - 1]; + + if (slowSCValues[i] != 0) { + scValues[i] = fastSCValues[i] / slowSCValues[i]; + } + + // Adaptive Moving Average (AMA) calculation + amaValues.Add(amaValues.Count == 0 ? prices[i] : amaValues[i - 1] + Math.Pow(scValues[i], 2) * (prices[i] - amaValues[i - 1]) / Math.Pow(1 + Math.Pow(scValues[i], 2), 0.5)); + } + + return amaValues; + } + // Hull Moving Average (HMA): + public static List calculate_HMA(List prices, int period) { + List wmaValues = calculate_HMA_wma_helper(prices, period); + List halfPeriodWmaValues = calculate_HMA_wma_helper(prices, period / 2); + + List sqrtPeriodWmaValues = new List(); + for (int i = 0; i < wmaValues.Count; i++) { + sqrtPeriodWmaValues.Add(Math.Sqrt(wmaValues[i])); + } + + List twicePeriodWmaValues = new List(); + for (int i = 0; i < halfPeriodWmaValues.Count; i++) { + twicePeriodWmaValues.Add(2 * halfPeriodWmaValues[i]); + } + + List hmaValues = new List(); + for (int i = 0; i < wmaValues.Count; i++) { + hmaValues.Add(twicePeriodWmaValues[i] * sqrtPeriodWmaValues[i]); + } + + return calculate_HMA_wma_helper(hmaValues, (int)Math.Sqrt(period)); + } + + static List calculate_HMA_wma_helper(List prices, int period) { + List wma_values = new List(); + + for (int i = 0; i < prices.Count; i++) { + double sum = 0; + double weightSum = 0; + + for (int j = Math.Max(0, i - period + 1); j <= i; j++) { + double weight = (j - i + period) * (j - i + period); + sum += prices[j] * weight; + weightSum += weight; + } + + if (weightSum != 0) { + wma_values.Add(sum / weightSum); + } else { + wma_values.Add(prices[i]); + } + } + + return wma_values; + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/EMAStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/EMAStream.cs new file mode 100644 index 000000000..1fa4bf6cf --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/EMAStream.cs @@ -0,0 +1,54 @@ + +using Google.Protobuf.WellKnownTypes; + +using System; +using BCG.Utilities.General; +using System.Data; +using bts.api.domain; + +namespace BCG { + // Stream computes the EMA of a column in any stream. It takes in the base stream, the column name and a smoothing factor + // as args, and create one row with an ema column, which is the running ema of the column provided + [DataStreamUserRespresentation("ema")] + public class EMAStream : ViewStream { + private ViewStream _stream; + private readonly string _col_name; + private EMACalculator _calc; + private readonly IBlueTableSchema _schema = BlueDataUtility.form_schema( + "SMA", Tuple.Create("id", TypeCode.Int32), + Tuple.Create("sma", TypeCode.Double)); + + public EMAStream(ViewStream stream, string col_name, double smoothing_factor) { + _col_name = col_name; + _stream = stream; + _calc = new EMACalculator(smoothing_factor); + } + + protected override void myOnFirstSubscriber() { + _stream.onChange += new ChangeDelegate(_onstream_change); + } + protected override void myOnLastSubscriber() { + _stream.onChange -= new ChangeDelegate(_onstream_change); + } + + + private void _onstream_change(object o) { + + DataPacket dp = o as DataPacket; + foreach (IBlueRow r in dp.addedRows) { + double price = r.get(_col_name); + if (!Double.IsNaN(price)) + _calc.add(price); + } + IBlueRow nr = new BlueRow(_schema, 0, _calc.get_EMA()); + BlueTable state = new BlueTable(_schema); + state.addRow(nr); + DataPacket dp_out = BlueDataUtility.build_dp_inc(state, myLastValue as DataPacket); + myFireEvent(dp_out); + } + } + +} + + + diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RenkoStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RenkoStream.cs new file mode 100644 index 000000000..c1506c268 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RenkoStream.cs @@ -0,0 +1,87 @@ + +using Google.Protobuf.WellKnownTypes; + +using System; +using BCG.Utilities.General; +using System.Data; +using bts.api.domain; +using System.Reactive.Concurrency; +using mbts.spark; + +namespace BCG { + // Stream computes the EMA of a column in any stream. It takes in the base stream, the column name and a smoothing factor + // as args, and create one row with an ema column, which is the running ema of the column provided + [DataStreamUserRespresentation("renko")] + public class RenkoStream : ViewStream { + private ViewStream _stream; + private readonly string _col_name; + private readonly double _brick_sz; + private RenkoCalculator _calc; + + private BlueTable _table = new BlueTable(BlueDataUtility.form_schema( + "Renko", Tuple.Create("id", TypeCode.Int32), + Tuple.Create("entry_price", TypeCode.Double), + Tuple.Create("current_price", TypeCode.Double), + Tuple.Create("brick_low", TypeCode.Double), + Tuple.Create("brick_high", TypeCode.Double), + Tuple.Create("brick_transition", TypeCode.String), + Tuple.Create("n_updates_within", TypeCode.Int32) + )); + + public RenkoStream(ViewStream stream, string col_name, double brick_sz) { + _col_name = col_name; + _stream = stream; + _brick_sz = brick_sz; + _calc = new RenkoCalculator(brick_sz); + } + + protected override void myOnFirstSubscriber() { + _stream.onChange += _onstream_change; + } + protected override void myOnLastSubscriber() { + _stream.onChange -= new ChangeDelegate(_onstream_change); + } + + + private IBlueRow _remove_row(RenkoBrick brick) { + var row = _table[brick.id]; + _table.deleteRow(row); + return row; + } + + private IBlueRow _create_new_row(RenkoBrick brick) { + return new BlueRow(_table.schema, brick.id, + brick.entry_price, brick.current_price, brick.low, brick.high, + brick.entry.ToString(), brick.n_updates); + } + + private void _onstream_change(object o) { + // what feeds in is a time series + BlueTable deleted_rows = new BlueTable(_table.schema); + BlueTable added_rows = new BlueTable(_table.schema); + DataPacket dp = o as DataPacket; + foreach (IBlueRow r in dp.addedRows) { + double val = r.get(_col_name); + var new_bricks = _calc.update(val); + if (new_bricks.Count() > 0) { + foreach (var nb in new_bricks) { + var new_row = _create_new_row(nb); + added_rows.addRow(new_row); + _table.addRow(new_row); + } + } else { + var existing_brick = _calc.current_brick; + var del_row = _remove_row(existing_brick); + deleted_rows.addRow(del_row); + var new_row = _create_new_row(existing_brick); + added_rows.addRow(new_row); + _table.addRow(new_row); + } + } + myFireEvent(new DataPacket(_table, added_rows, deleted_rows)); + } + } +} + + + diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SMAStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SMAStream.cs new file mode 100644 index 000000000..3bea28f4f --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SMAStream.cs @@ -0,0 +1,54 @@ + +using Google.Protobuf.WellKnownTypes; + +using System; +using BCG.Utilities.General; +using System.Data; +using bts.api.domain; + +namespace BCG { + // Stream computes the SMA of a column in any stream. It takes in the base stream, the column namer and a smoothing factor + // as args, and create one row with an sma column, which is the running ema of the column provided + [DataStreamUserRespresentation("sma")] + public class SMAStream : ViewStream { + private ViewStream _stream; + private readonly string _col_name; + private Chartutils.SMACalculator _calc; + private readonly IBlueTableSchema _schema = BlueDataUtility.form_schema( + "SMA", Tuple.Create("id", TypeCode.Int32), + Tuple.Create("sma", TypeCode.Double)); + + public SMAStream(ViewStream stream, string col_name, int window_size) { + _col_name = col_name; + _stream = stream; + _calc = new Chartutils.SMACalculator(window_size); + } + + protected override void myOnFirstSubscriber() { + _stream.onChange += new ChangeDelegate(_onstream_change); + } + protected override void myOnLastSubscriber() { + _stream.onChange -= new ChangeDelegate(_onstream_change); + } + + + private void _onstream_change(object o) { + + DataPacket dp = o as DataPacket; + foreach (IBlueRow r in dp.addedRows) { + double price = r.get(_col_name); + if (!Double.IsNaN(price)) + _calc.add(r.get(_col_name)); + } + IBlueRow nr = new BlueRow(_schema, 0, _calc.get_SMA()); + BlueTable state = new BlueTable(_schema); + state.addRow(nr); + DataPacket dp_out = BlueDataUtility.build_dp_inc(state, myLastValue as DataPacket); + myFireEvent(dp_out); + } + } + +} + + + diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimeSalesStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimeSalesStream.cs index acad6ab0d..153da7e76 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimeSalesStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimeSalesStream.cs @@ -1,5 +1,4 @@ -namespace BCG -{ +namespace BCG { using BCG.Utilities.General; using bts.api; @@ -18,55 +17,54 @@ namespace BCG using Spark.Api.MarketData; using SecurityKey = System.UInt32; + using bts; [DataStreamUserRespresentation("timesales")] - public class TimeSalesStream : ViewStream, IWorker - { + public class TimeSalesStream : ViewStream, IWorker { private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500); private DateTime _last_poll_time = DateTime.MinValue; private ViewStreamDatabase _vsd; private int _id = 0; - IReadOnlyDictionary _cached_state = null; + //IReadOnlyDictionary _cached_state = null; MarketDataClient _md_client; private bool _dirty = false; private ConcurrentQueue _cached_trades = new ConcurrentQueue(); + private CircularRowCollection _circulare_table; private readonly int _buffer_size; private IBlueTableSchema _schema = BlueDataUtility.form_schema("TimeSales", Tuple.Create("id", TypeCode.Int32), Tuple.Create("sec_key", TypeCode.UInt32), Tuple.Create("sec", TypeCode.String), Tuple.Create("time", TypeCode.DateTime), + Tuple.Create("time_ll", TypeCode.Int64), Tuple.Create("price", TypeCode.Double), Tuple.Create("qty", TypeCode.UInt32), Tuple.Create("aggressor", TypeCode.String)); public TimeSalesStream(int buffer_size) - : this(buffer_size, ViewStreamDatabase.instance) - { + : this(buffer_size, ViewStreamDatabase.instance) { + } - public TimeSalesStream(int buffer_size, ViewStreamDatabase vsd) - { + public TimeSalesStream(int buffer_size, ViewStreamDatabase vsd) { _buffer_size = buffer_size; + _circulare_table = new CircularRowCollection(_schema, buffer_size); bts.Log.Gen.info("TimeSalesStream.cons"); _vsd = vsd; _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5))); } - protected override void myOnFirstSubscriber() - { + protected override void myOnFirstSubscriber() { _vsd.addWorker(this); _md_client.trades(new TradesSubscriptionFilter()).Subscribe(_on_trades); //_md_client.trades(null).Subscribe(_on_trades); } - private void _log_updates(IList trades) - { + private void _log_updates(IList trades) { bts.Log.Gen.info($"TimeSalesStream update sz: {trades.Count}"); } - private void _on_trades(IList trades) - { + private void _on_trades(IList trades) { _dirty = true; _log_updates(trades); // just cacching in here, will populate the rows in _poll_scan and fire the datapacket @@ -75,35 +73,28 @@ namespace BCG _cached_trades.Enqueue(t); } - protected override void myOnLastSubscriber() - { + protected override void myOnLastSubscriber() { _vsd.removeWorker(this); } - private void _scan_trades() - { - Func to_dt = (tinf) => - { + private void _scan_trades() { + Func to_dt = (tinf) => { return new mbts.utils.LLDateTime((ulong)tinf.trade.Time).to_datetime().ToLocalTime(); }; - List trades = new List(_cached_trades); - var latest_n_trades = trades.OrderByDescending(t => (ulong)t.trade.Time).Take(_buffer_size).ToList(); - BlueTable new_state = new BlueTable(_schema); - foreach (TradeInfo tinf in Enumerable.Reverse(latest_n_trades)) - { - // TODO, time - IBlueRow r = new BlueRow(_schema, _id++, tinf.security.Id, tinf.security.ToString(), to_dt(tinf), tinf.trade.Mantissa * Math.Pow(10, tinf.trade.Exponent), tinf.trade.Size, tinf.trade.AggressorSide.ToString()); - new_state.addRow(r); + bts.Log.Gen.info($"TimeSales stream polling {trades.Count} trades"); + // OrderByDescending == (high to low) newest on top + var sorted_latest_trades = trades.OrderByDescending(t => (ulong)t.trade.Time);//.Take(_buffer_size).ToList(); + foreach (TradeInfo tinf in /*Enumerable.Reverse(latest_n_trades)*/sorted_latest_trades) { + IBlueRow r = new BlueRow(_schema, _id++, tinf.security.Id, tinf.security.ToString(), to_dt(tinf), tinf.trade.Time, tinf.trade.Mantissa * Math.Pow(10, tinf.trade.Exponent), tinf.trade.Size, tinf.trade.AggressorSide.ToString()); + _circulare_table.addRow(r); } - DataPacket dp = BlueDataUtility.build_dp_inc(new_state, myLastValue as DataPacket); + DataPacket dp = BlueDataUtility.build_dp_inc(_circulare_table, myLastValue as DataPacket); myFireEvent(dp); } - public void doWork() - { - if (_dirty && DateTime.Now - _last_poll_time > _poll_period) - { + public void doWork() { + if (_dirty && DateTime.Now - _last_poll_time > _poll_period) { _last_poll_time = DateTime.Now; _dirty = false; _scan_trades(); diff --git a/src/lib/cmake/GetWindowsClientLinkLibraries.cmake b/src/lib/cmake/GetWindowsClientLinkLibraries.cmake index 3952a2895..ae1594222 100644 --- a/src/lib/cmake/GetWindowsClientLinkLibraries.cmake +++ b/src/lib/cmake/GetWindowsClientLinkLibraries.cmake @@ -2,7 +2,7 @@ function(get_link_libraries dest) find_package(fmt CONFIG REQUIRED) set(${dest} crypt32 - fmt::fmt + fmt::fmt-header-only libcrypto libcurl libboost_system