diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/NullFilterStream.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/NullFilterStream.cs index 62feaf799..366114bfe 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Filter/NullFilterStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/NullFilterStream.cs @@ -4,6 +4,8 @@ namespace bts.SilverPine { using System; public class NullFilterStream : ViewStream { + public NullFilterStream(ViewStream parent_str, string col_name) + : this(parent_str, col_name, true) { } public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) { ViewStream rs = new FilterStream(parent_str, r => { diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByBundle.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByBundle.cs new file mode 100644 index 000000000..f02b21ed6 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByBundle.cs @@ -0,0 +1,91 @@ +namespace bts.SilverPine { + using BCG; + + using System; + using System.Collections.Generic; + using System.Linq; + + using ZstdSharp.Unsafe; + + public enum GroupByBundleOp { + Mean, + Median, + Mode, + StdDev, + Max, + Min, + First, + Last, + Sum + } + public class GroupByBundle { + public readonly List> operations; + + public static GroupByBundle parse(string text) { + // text is inp_col: op, inp_col: op, inp_col:op + List> rv = new List>(); + string[] calcs = text.Split(','); + foreach (string c in calcs) { + string[] toks = c.Split(':'); + string inp_col = toks[0].Trim(); + string op_txt = toks[1].Trim(); + GroupByBundleOp op = Enum.Parse(op_txt, true); + rv.Add(Tuple.Create(inp_col, op)); + } + return new GroupByBundle(rv); + } + + public GroupByBundle(List> operations_) { + operations = operations_; + } + + public static IBlueRow group_by(List rows, GroupByBundle parameters) { + if (rows.Count() == 0) { + return null; + } + + IBlueRow ir = rows.ElementAt(0); + IBlueTableSchema in_schema = ir.schema; + IBlueTableSchema out_schema = in_schema; + IDictionary> out_col_map = + new Dictionary>(); + foreach (var t in parameters.operations) { + string out_col_name = $"{t.Item1}_{t.Item2}".ToLower(); + out_schema = out_schema.add_column(TypeCode.Double, out_col_name); + out_col_map[out_col_name] = t; + } + MutableBlueRow or = new MutableBlueRow(out_schema, ir); + foreach (var kv in out_col_map) { + string out_col_name = kv.Key; + string in_col_name = kv.Value.Item1; + GroupByBundleOp op = kv.Value.Item2; + IEnumerable col_vals = BlueDataUtility.get_colvalues(rows, in_col_name); + double comp_res = compute_op(col_vals, op); + or.setValue(out_col_name, comp_res); + } + return or; + } + + public static double compute_op(IEnumerable col_vals, GroupByBundleOp op) { + switch (op) { + case GroupByBundleOp.Max: + return col_vals.Max(); + case GroupByBundleOp.Min: + return col_vals.Min(); + case GroupByBundleOp.Mean: + return col_vals.Sum() / col_vals.Count(); + case GroupByBundleOp.Sum: + return col_vals.Sum(); + case GroupByBundleOp.First: + return col_vals.First(); + case GroupByBundleOp.Last: + return col_vals.Last(); + default: + // TODO + Log.assert(false); + return Double.NaN; + } + } + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs index c8b60b508..630faf094 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs @@ -2,10 +2,13 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Text.Json; using bts; +using bts.SilverPine; namespace BCG { + [DataStreamUserRespresentation("aggregate")] public class NonIncGroupByStream : ViewStream { private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator { Func _groupby_cols_func; @@ -43,13 +46,31 @@ namespace BCG { _main_stream = main_stream; _groupby_func = groupby_func; _groupby_cols = _groupby_func.get_group_by_cols(); - _row_hasher = new BlueRowHasher(_groupby_cols.Length); + _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length); } public NonIncGroupByStream(ViewStream main_stream, Func groupby_cols_func, Func, List> evaluate_func) : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { } + public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) { + GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text); + Func groupby_cols_func = () => { + return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col }; + }; + Func, List> evaluate_func = (group_rows) => { + if (!String.IsNullOrEmpty(optional_sort_col)) + group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col)); + IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by); + return new IBlueRow[] { rv }.ToList(); + }; + ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func); + mySetRealSource(rs); + } + public NonIncGroupByStream(ViewStream parent_str, string standard_group_by_text) : + this(parent_str, String.Empty, standard_group_by_text) { + } + protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; } @@ -79,7 +100,9 @@ namespace BCG { Log.DataFlow.info( $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}"); } - myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows)); + // TODO, + if (total_state != null) + myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows)); } } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs index ef5ad6180..8f7bb53db 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs @@ -1,10 +1,12 @@ using bts; +using bts.collections; using mbts; using System; using System.Collections.Generic; using System.Diagnostics; +using System.Linq; using System.Security.Cryptography; namespace BCG { @@ -35,6 +37,38 @@ namespace BCG { _is_tag_along = _join_spec.join_col_names.Length == 0; } + private bool _filled_missing_data = false; + private void _fill_missing_data(DataPacket dp) { + if (_filled_missing_data) + return; + Func null_or_empty = (str_arr) => { + if (str_arr == null || str_arr.Length == 0) + return true; + foreach (string s in str_arr) { + if (!String.IsNullOrEmpty(s)) { + return false; + } + } + return true; + }; + Func remove_bad_cols = (col_names) => { + HashSet schema_col_map = new HashSet(dp.schema.columnNames); + List cols_list = new List(col_names); + cols_list.RemoveWhere(c => !schema_col_map.Contains(c)); + return cols_list.ToArray(); + }; + if (null_or_empty(_join_spec.data_col_names)) { + _join_spec.data_col_names = dp.schema.columnNames; + } + _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names); + if (null_or_empty(_join_spec.data_col_names)) { + _join_spec.data_col_names = dp.schema.columnNames; + } + if (null_or_empty(_join_spec.projected_col_names)) { + _join_spec.projected_col_names = _join_spec.data_col_names; + } + } + public void init() { _join_spec.stream.onChange += _on_stream_cb; } @@ -43,6 +77,7 @@ namespace BCG { Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}"); var timer = Stopwatch.StartNew(); DataPacket dp = (DataPacket)dp_obj; + _fill_missing_data(dp); if (_schema == null) { _schema = dp.schema; _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names); diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs index 092152d22..432dbd539 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs @@ -7,11 +7,12 @@ using System.Threading.Tasks; using mbts; namespace BCG { + // this a collection of data, theres no joining logic in here public class NonIncJoinSpec { public ViewStream stream; // join col names are the names on which the streams are joined (for equality), // data col names are the names in the original stream - // projected col naes are their new names, ie if its x:l_x, then x is in the + // projected col names are their new names, ie if its x:l_x, then x is in the // data_col_names and l_x in projected cols public string[] join_col_names, data_col_names, projected_col_names; diff --git a/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs index 818f3e60c..9300efb1d 100644 --- a/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs +++ b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs @@ -9,161 +9,16 @@ using BCG; using System.Linq; using System.Data.Entity; using System.Collections.Generic; +using System.Data.SQLite; + namespace bts.SilverPine { - public class DataRowWrapperRow : IBlueRow { - private readonly DataRow _row; - private readonly IBlueTableSchema _schema; - private readonly int _id; - public DataRowWrapperRow(IBlueTableSchema schema, DataRow row, int id) { - _row = row; - _id = id; - _schema = schema; - } - public virtual IBlueTableSchema schema { get { return _schema; } } - - public virtual object this[string col_name] { - get { - ColumnInfo ci = _schema[col_name]; - return this[ci.colNo]; - } - } - public virtual object this[int index] { - get { - if (_schema[index].colName == "id") - return _id; - return _row[index]; - } - } - - public virtual int Count { get { return _row.ItemArray.Length; } } - - public virtual bool IsSynchronized { get { return false; } } - - public virtual object SyncRoot { get { return this; } } - - public virtual void CopyTo(Array array, int index) { _row.ItemArray.CopyTo(array, index); } - - public virtual bool IsFixedSize { get { return true; } } - - virtual public bool IsReadOnly { get { return true; } } - - /// - /// The key can either be the string corrsponding to the column name or - /// the order of the column - /// - public virtual object this[object key] { - get { - if (key is string) - return this[(string)key]; - else if (key is int) - return this[(int)key]; - throw new BlueDataException("Wrong type while indexing the row"); - } - set { throw new NotSupportedException("The IBlueRow is readonly"); } - } - - public virtual ICollection Keys { get { return _schema.columnNames; } } - - public virtual ICollection Values { get { return _row.ItemArray; } } - - public virtual void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); } - - public virtual void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); } - - public virtual bool Contains(object key) { return _schema[(string)key] != null; } - - public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } - - IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); } - - public virtual void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); } - - } - - public class DataTableWrapperTable : IBlueRowCollection { - - public class DataTableWrapperTableEnumerator : IEnumerator { - private IEnumerator _enumerator; - private IBlueTableSchema _schema; - private int _id = 0; - public DataTableWrapperTableEnumerator(DataTable table) { - _enumerator = table.Rows.GetEnumerator(); - _schema = BlueDataUtility.getSchema(table); - - } - public object Current { - get { - DataRow r = _enumerator.Current as DataRow; - return new DataRowWrapperRow(_schema, r, _id); - } - } - public bool MoveNext() { - _id++; - return _enumerator.MoveNext(); - } - public void Reset() { - _id = 0; - _enumerator.Reset(); - } - } - - private readonly DataTable _table; - private IBlueTableSchema _schema; - private IEnumerator _enumerator; - - public DataTableWrapperTable(DataTable table) { - _table = table; - _schema = BlueDataUtility.getSchema(_table); - _enumerator = new DataTableWrapperTableEnumerator(table); - } - public virtual IBlueRow this[int id] { - get { - // its assumed the rows index == id - DataRow dr = _table.Rows[id]; - return new DataRowWrapperRow(_schema, dr, id); - } - set { - - } - } - - public virtual IBlueTableSchema schema { get { return _schema; } } - - public virtual bool containsID(int id) { return id < _table.Rows.Count; } - - // private List _rows() { - // List rv = new List(); - // int id = 0; - // foreach (DataRow r in _table.Rows) { - // rv.Add(new DataRowWrapperRow(r, id++)); - // } - // return rv; - // } - - public virtual IEnumerator GetEnumerator() { - // var rows = _rows(); - // return rows.GetEnumerator(); - return _enumerator; - } - - public virtual bool IsSynchronized { get { return false; } } - - public virtual object SyncRoot { get { return this; } } - - public virtual void CopyTo(Array array, int index) { - Log.assert(false); - //_rows().CopyTo(array as IBlueRow[], index); - } - - public virtual int Count { - get { return _table.Rows.Count; } - } - } - public static class SQLiteHelper { - public static DataTable ExecuteSelectQuery(string connection_string, string query) { + public static string location = "C:\\Users\\m_tah\\development\\data\\"; + public static string db_name = "stream_data.db"; + + public static DataTable execute_select_query(string connection_string, string query) { // Create a new DataTable to store the results DataTable dataTable = new DataTable(); @@ -183,48 +38,85 @@ namespace bts.SilverPine { Console.WriteLine("An error occurred: " + ex.Message); } } - // Return the populated DataTable return dataTable; } - } - [DataStreamUserRespresentation("SQLite")] - public class SQLiteStream : ViewStream { - private readonly string _connection_string, _query; - private Stopwatch _stopwatch = new Stopwatch(); - public SQLiteStream(string connection_string, string query) { - _connection_string = connection_string; - _query = query; + public static List> ExecuteBulkRead(string connectionString, string query) { + var result = new List>(); + + using (var connection = new SQLiteConnection(connectionString)) { + connection.Open(); + + // Begin a transaction to optimize read speed + using (var transaction = connection.BeginTransaction()) + using (var command = new SQLiteCommand(query, connection, transaction)) + using (var reader = command.ExecuteReader()) { + while (reader.Read()) { + var row = new Dictionary(); + + for (int i = 0; i < reader.FieldCount; i++) { + string columnName = reader.GetName(i); + object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i); + row[columnName] = columnValue; + } + + result.Add(row); + } + + // Commit the transaction + transaction.Commit(); + } + } + + return result; } + public static List> get_table(string connectionString, string query) { + var result = new List>(); - private void _read_from_sqlite_nd_fire() { - _stopwatch.Start(); - DataTable dt = SQLiteHelper.ExecuteSelectQuery(_connection_string, _query); - _stopwatch.Stop(); - Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to read db: {_stopwatch.ElapsedMilliseconds} msecs"); - _stopwatch.Start(); - IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt); - //IBlueRowCollection rows = new DataTableWrapperTable(dt); - _stopwatch.Stop(); - Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs"); - DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket); - myFireEvent(dp); + using (var connection = new SQLiteConnection(connectionString)) { + try { + connection.Open(); + + using (var command = new SQLiteCommand(query, connection)) + using (var reader = command.ExecuteReader()) { + while (reader.Read()) { + var row = new Dictionary(); + + for (int i = 0; i < reader.FieldCount; i++) { + string columnName = reader.GetName(i); + object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i); + row[columnName] = columnValue; + } + + result.Add(row); + } + } + } catch (Exception ex) { + Console.WriteLine("An error occurred: " + ex.Message); + } + } + return result; } - protected override void myOnFirstSubscriber() { - _read_from_sqlite_nd_fire(); + public static IEnumerable> get_table_2(string connectionString, string query) { + using (var connection = new SQLiteConnection(connectionString)) { + connection.Open(); + using (var command = new SQLiteCommand(query, connection)) + using (var reader = command.ExecuteReader()) { + while (reader.Read()) { + var row = new Dictionary(); + for (int i = 0; i < reader.FieldCount; i++) { + string columnName = reader.GetName(i); + object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i); + row[columnName] = columnValue; + } + yield return row; + } + } + } } - - protected override void myOnLastSubscriber() { - // this value will be cleared by the base class - // but in this particular stream this behaviour - // is not usefull since any new subscribers - // would force the query to be recreated - //myLastDP = myLastValue; - } - } } diff --git a/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SqliteStream.cs b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SqliteStream.cs new file mode 100644 index 000000000..0dc15e768 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SqliteStream.cs @@ -0,0 +1,73 @@ +namespace bts.SilverPine { + using System; + using System.Data; + using System.Data.SQLite; + using System.Diagnostics; + using System.Collections; + + + using BCG; + + [DataStreamUserRespresentation("SQLite")] + public class SQLiteStream : ViewStream { + private readonly string _connection_string, _query; + private Stopwatch _stopwatch = new Stopwatch(); + + private SQLiteStream(string connection_string, string query, bool noop) { + _connection_string = connection_string; + _query = query; + } + + public SQLiteStream(string db_name, string query) + : + this($"Data Source={SQLiteHelper.location}{db_name};Version=3", query, true) { } + + public SQLiteStream(string query) + : + this(SQLiteHelper.db_name, query) { + } + + + private void _read_from_sqlite_and_fire() { + _stopwatch.Start(); + DataTable dt = SQLiteHelper.execute_select_query(_connection_string, _query); + _stopwatch.Stop(); + Log.Gen.info($"SQLiteStream.ExecuteSelectQuery time to read db: {_stopwatch.ElapsedMilliseconds} msecs"); + _stopwatch.Restart(); + #region perf measures + // SQLiteHelper.get_table(_connection_string, _query); + // _stopwatch.Stop(); + // Log.Gen.info($"SQLiteStream.get_table time to read db: {_stopwatch.ElapsedMilliseconds} msecs"); + // _stopwatch.Restart(); + // SQLiteHelper.ExecuteBulkRead(_connection_string, _query); + // _stopwatch.Stop(); + // Log.Gen.info($"SQLiteStream.ExecuteBulkRead time to read db: {_stopwatch.ElapsedMilliseconds} msecs"); + // _stopwatch.Restart(); + // SQLiteHelper.get_table_2(_connection_string, _query); + // _stopwatch.Stop(); + // Log.Gen.info($"SQLiteStream.get_table_2 time to read db: {_stopwatch.ElapsedMilliseconds} msecs"); + // _stopwatch.Restart(); + #endregion + IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt); + //IBlueRowCollection rows = new DataTableWrapperTable(dt); + _stopwatch.Stop(); + Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs"); + DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket); + myFireEvent(dp); + } + + protected override void myOnFirstSubscriber() { + _read_from_sqlite_and_fire(); + } + + protected override void myOnLastSubscriber() { + // this value will be cleared by the base class + // but in this particular stream this behaviour + // is not usefull since any new subscribers + // would force the query to be recreated + //myLastDP = myLastValue; + } + + } + +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs index 6c68fd2dd..24e6facb5 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs @@ -80,7 +80,7 @@ namespace BCG { public IBlueRow map(IBlueRow row) { MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row); - rv.setValue(myConversionColName, mapDateTime(row[myConversionColName], myConversion)); + rv.setValue(myConversionColName, mapDateTime(row.get_datetime(myConversionColName), myConversion)); return rv; //return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName); } diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/DateTimeExpanstionStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/DateTimeExpanstionStream.cs index 03a4ec900..a2bfb548a 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Time/DateTimeExpanstionStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Time/DateTimeExpanstionStream.cs @@ -10,9 +10,11 @@ namespace bts.SilverPine { Func get_schema = (s) => { IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date"); rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day"); - rv = rv.add_column(System.TypeCode.String, $"{inp_col}_month"); - rv = rv.add_column(System.TypeCode.String, $"{inp_col}_hour"); - rv = rv.add_column(System.TypeCode.String, $"{inp_col}_min"); + rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month"); + rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month"); + rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour"); + rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_min"); + rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day"); return rv; }; ViewStream rs = new MapStream(parent_stream, @@ -23,9 +25,11 @@ namespace bts.SilverPine { MutableBlueRow rv = new MutableBlueRow(s, r); rv.setValue($"{inp_col}_date", dt.Date); rv.setValue($"{inp_col}_day", dt.DayOfWeek.ToString()); - rv.setValue($"{inp_col}_month", dt.Date.Month.ToString()); + rv.setValue($"{inp_col}_day_of_month", dt.Day); + rv.setValue($"{inp_col}_month", dt.Date.Month); rv.setValue($"{inp_col}_hour", dt.Hour); rv.setValue($"{inp_col}_min", dt.Minute); + rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds); return rv; }); mySetRealSource(rs); diff --git a/src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs b/src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs index 90e660f22..afb888cfa 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs @@ -39,7 +39,7 @@ namespace bts.SilverPine { mySetRealSource(rs); } - // for each row, does a row[new_ol_name] = row[col1] - rows[row_ffset][col2] + // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2] // we should generlize this for any math operation public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) { // use GlobalMapStream @@ -91,19 +91,32 @@ namespace bts.SilverPine { DataPacket dp = dp_obj as DataPacket; List rows = dp.newRows.to_list(); var f1_rows = rows.Where(r => _f1(r)); - Log.assert(f1_rows.Count() == 1); + //Log.assert(f1_rows.Count() == 1); var f2_rows = rows.Where(r => _f2(r)); - Log.assert(f2_rows.Count() == 1); - IBlueRow f1_r = f1_rows.ElementAt(0); - IBlueRow f2_r = f2_rows.ElementAt(0); - double diff = f1_r.get(_c1) - f2_r.get(_c2); - IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff")); - BlueTable out_table = new BlueTable(out_schema); - MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r); - out_row.setValue("diff", diff); - out_table.addRow(out_row); - DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket); - myFireEvent(dp_out); + //Log.assert(f2_rows.Count() == 1); + if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) { + IBlueRow f1_r = f1_rows.ElementAt(0); + IBlueRow f2_r = f2_rows.ElementAt(0); + double diff = f1_r.get(_c1) - f2_r.get(_c2); + IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff")); + BlueTable out_table = new BlueTable(out_schema); + MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r); + out_row.setValue("diff", diff); + out_table.addRow(out_row); + DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket); + myFireEvent(dp_out); + } else { + IBlueRow f1_r = rows.ElementAt(0); + IBlueRow f2_r = rows.ElementAt(0); + double diff = Double.NaN; + IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff")); + BlueTable out_table = new BlueTable(out_schema); + MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r); + out_row.setValue("diff", diff); + out_table.addRow(out_row); + DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket); + myFireEvent(dp_out); + } } } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs index ef9f2499f..5a62f9c4a 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs @@ -18,6 +18,7 @@ using System.Text.Json.Nodes; using Newtonsoft.Json; using BCG; using System.Runtime.CompilerServices; +using bts; namespace bts { @@ -207,6 +208,10 @@ namespace BCG { return rv; } + public static bool contains_column(this IBlueTableSchema input_schema, string col) { + return input_schema.columnNames.Contains(col); + } + public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple[] cols) { List cis = new List(input_schema.columnInfo); foreach (Tuple c in cols) @@ -1719,8 +1724,10 @@ namespace BCG { public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) { List rv = new List(); - foreach (string c in col_names) + foreach (string c in col_names) { + Log.assert(schema.contains_column(c), $"Could not find col: {c} in table: {schema.tableName}"); rv.Add(schema[c].colNo); + } return rv.ToArray(); } @@ -2207,23 +2214,53 @@ namespace BCG { try { if (v == null || v == DBNull.Value) return error_val; - else { - if (v is DateTime) - return (DateTime)v; - else if (v is string) { - DateTime rv_dt = error_val; - DateTime.TryParse((string)v, out rv_dt); + + if (v is DateTime) { + return (DateTime)v; + } else if (v is string) { + DateTime rv_dt; + + // Try parsing the string as a DateTime + if (DateTime.TryParse((string)v, + System.Globalization.CultureInfo.InvariantCulture, + System.Globalization.DateTimeStyles.RoundtripKind, + out rv_dt)) { return rv_dt; } else { - double oa_date = (double)Convert.ChangeType(v, typeof(Double)); - return DateTime.FromOADate(oa_date); + return error_val; // Return the error value if parsing fails } + } else { + // Assume the input can be converted to a double and use FromOADate + double oa_date = (double)Convert.ChangeType(v, typeof(Double)); + return DateTime.FromOADate(oa_date); } } catch { return error_val; } } + + // private static DateTime _get_datetime_value(object v, DateTime error_val) { + // try { + // if (v == null || v == DBNull.Value) + // return error_val; + // else { + // if (v is DateTime) + // return (DateTime)v; + // else if (v is string) { + // DateTime rv_dt = error_val; + // DateTime.TryParse((string)v, out rv_dt); + // return rv_dt; + // } else { + // double oa_date = (double)Convert.ChangeType(v, typeof(Double)); + // return DateTime.FromOADate(oa_date); + // } + // } + // } catch { + // return error_val; + // } + // } + #endregion #region CSV methods diff --git a/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs b/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs index 7ffefa52a..5c3f0e389 100644 --- a/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs +++ b/src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs @@ -47,6 +47,11 @@ namespace BCG { /// /// public void loadAssembly(string assemblyName) { + Func is_silverpine_only = (t) => { + if (!assemblyName.Contains("SharedStreams.dll")) + return true; + return t.Namespace == "bts.SilverPine"; + }; bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}"); if (myAssemblyNames.Contains(assemblyName)) return; @@ -56,7 +61,7 @@ namespace BCG { Assembly.LoadFrom(assemblyName); foreach (Type type in loadedAssembly.GetTypes()) - if (!type.IsInterface + if (!type.IsInterface && is_silverpine_only(type) && !type.IsAbstract && type.IsSubclassOf(typeof(ViewStream)) && type.IsPublic) { diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json b/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json index 7154e2423..f56f3ec5f 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json @@ -1,60 +1,62 @@ -"user": "bts", -"pubsub": { - "milliseconds_between_sub_info_updates": 1000, - "milliseconds_between_time_slice_updates": 5000, - "milliseconds_between_no_buffer_space_retries": 100, - "retries_on_no_buffer_space": 1, - "do_it_all_seconds_between_logs": 0, - "heartbeat_interval": 15000, - "missing_heartbeats_before_gc": 50, - "use_managed_sockets": true, - "state_update_period": 5000, - "statistics_update_period": 5000, - "disposed_connection_check_period": 1000, - "message_processing_warning_threshold": 1000, - "server_port": 12345, - "publisher_routes": [ - { - "name": "default", - "server_address": "bts-cloud://192.168.1.80:12345", - "tag": ".*" - } - ], - "subscriber_routes": [ - { - "name": "default", - "server_address": "bts-cloud://192.168.1.80:12345", - "tag": ".*" - } - ], - "max_packet_size": 500000000, - "blue_comm_must_die": true, - "crushable_tags": [], - "tag_type_to_assembly_map": [ - { - "handler_class": "BCG.PubSubServerTagHandler", - "assembly": "DataStreamCore.dll" - } - ], - "server_queue": "PubSubQueue", - "port_range": { - "first": 30400, - "last": 31700 - }, - "bind_to_random_port": false, - "debug_tags": "^NOTAGS$", - "latency_report_threshold_ms": 1500, - "use_substations": false, - "use_dynamic_clients": false, - "is_substation": false, - "internally_publish_sub_info": true, - "use_managed_clients_in_native_code": false, - "compressed_tags": "^NOTAGS$", - "tags_to_report_compression_ratio": "^NOTAGS$", - "disable_msmq": false, - "use_prepublished_server_state": false, - "ip_translation": null, - "gc_activation_threshold": 0, - "wait_period_after_gc": 1000 -} +{ + "user": "bts", + "sparkd": "http://192.168.1.80:8000/config", + "pubsub": { + "milliseconds_between_sub_info_updates": 1000, + "milliseconds_between_time_slice_updates": 5000, + "milliseconds_between_no_buffer_space_retries": 100, + "retries_on_no_buffer_space": 1, + "do_it_all_seconds_between_logs": 0, + "heartbeat_interval": 15000, + "missing_heartbeats_before_gc": 50, + "use_managed_sockets": true, + "state_update_period": 5000, + "statistics_update_period": 5000, + "disposed_connection_check_period": 1000, + "message_processing_warning_threshold": 1000, + "server_port": 12345, + "publisher_routes": [ + { + "name": "default", + "server_address": "bts-cloud://192.168.1.80:12345", + "tag": ".*" + } + ], + "subscriber_routes": [ + { + "name": "default", + "server_address": "bts-cloud://192.168.1.80:12345", + "tag": ".*" + } + ], + "max_packet_size": 500000000, + "blue_comm_must_die": true, + "crushable_tags": [], + "tag_type_to_assembly_map": [ + { + "handler_class": "BCG.PubSubServerTagHandler", + "assembly": "DataStreamCore.dll" + } + ], + "server_queue": "PubSubQueue", + "port_range": { + "first": 30400, + "last": 31700 + }, + "bind_to_random_port": false, + "debug_tags": "^NOTAGS$", + "latency_report_threshold_ms": 1500, + "use_substations": false, + "use_dynamic_clients": false, + "is_substation": false, + "internally_publish_sub_info": true, + "use_managed_clients_in_native_code": false, + "compressed_tags": "^NOTAGS$", + "tags_to_report_compression_ratio": "^NOTAGS$", + "disable_msmq": false, + "use_prepublished_server_state": false, + "ip_translation": null, + "gc_activation_threshold": 0, + "wait_period_after_gc": 1000 + } } \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs b/src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs index 22f35d514..91bb628d2 100644 --- a/src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs +++ b/src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs @@ -9,8 +9,90 @@ namespace bts.SilverPine { using ZstdSharp.Unsafe; public static class ContractUtils { + public class FutureTradableContract { + public string symbol; + public DateTime expiration; + public Exchange exchange; + public string contract_text; + } + public class FutureSpreadContract { + public List> legs; + public string symbol { + get { + return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol; + } + } + public string contract_text; + } + public class FutureContinousContract { + public string symbol; + public Exchange exchange; + } + // this is for all future conytracts belonging to a symbol + public class FutureWildcardContract { + public string symbol; + public Exchange exchange; + } + public class FutureContract { + public FutureWildcardContract wildcard_contract = null; + public FutureTradableContract future_tradable_contract = null; + public FutureSpreadContract spread_tradable_contract = null; + public FutureContinousContract continous_constract = null; + + public FutureContract(string txt) { + wildcard_contract = parse_wildcard_contract(txt); + if (wildcard_contract != null) + return; + FutureTradableContract fc = parse_future_contract(txt); + if (fc != null) { + future_tradable_contract = fc; + return; + } + FutureSpreadContract sc = parse_future_spread_contract(txt); + if (sc != null) { + spread_tradable_contract = sc; + return; + } + // this is not totally true, since ParseFutureSpreadContract does not + // parse ICS spreads + continous_constract = new FutureContinousContract() { symbol = txt }; + } + public bool is_continous_contract => continous_constract != null; + public string symbol() { + if (wildcard_contract != null) + return wildcard_contract.symbol; + if (continous_constract != null) { + return continous_constract.symbol; + } else if (future_tradable_contract != null) { + return future_tradable_contract.symbol; + } else if (spread_tradable_contract != null) { + return spread_tradable_contract.symbol; + } + Log.assert(false); + return String.Empty; + } + public string contract_text { + get { + if (future_tradable_contract != null) + return future_tradable_contract.contract_text; + if (spread_tradable_contract != null) + return spread_tradable_contract.contract_text; + return null; + } + } + } #region Parsing - public static FutureContract ParseFutureContract(string contract_txt, Exchange exchange = Exchange.CME) { + public static FutureWildcardContract parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) { + if (contract_txt.EndsWith("*")) { + contract_txt = contract_txt.Replace("*", String.Empty); + return new FutureWildcardContract() { + symbol = contract_txt, + exchange = exchange + }; + } + return null; + } + public static FutureTradableContract parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) { // Validate the future string length if (!contract_txt.All(c => Char.IsLetterOrDigit(c))) return null; @@ -56,23 +138,24 @@ namespace bts.SilverPine { // Create the FutureContract instance and set its properties DateTime expiration = new DateTime(year, month, 1); - var futureContract = new FutureContract { + var futureContract = new FutureTradableContract { symbol = symbol, expiration = expiration, - exchange = exchange + exchange = exchange, + contract_text = contract_txt }; return futureContract; } // Method to parse a futures spread, e.g. "CLX4-CLZ4" - public static FutureSpreadContract ParseFutureSpreadContract(string spreadString, Exchange exchange = Exchange.CME) { + public static FutureSpreadContract parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) { // Split the string by '-' to handle each leg - var spreadParts = spreadString.Split('-'); + var spreadParts = contract_txt.Split('-'); if (spreadParts.Length < 2) { return null; // Not a valid spread format } - var spreadContract = new FutureSpreadContract { legs = new List>() }; + var spreadContract = new FutureSpreadContract { legs = new List>() }; foreach (var part in spreadParts) { // Split each leg by '*', assuming the format is weight*future or just future @@ -92,33 +175,32 @@ namespace bts.SilverPine { } // Parse the future contract - var futureContract = ParseFutureContract(futurePart, exchange); + var futureContract = parse_future_contract(futurePart, exchange); if (futureContract == null) { return null; // Failed to parse the future part } // Add the leg to the spread contract - spreadContract.legs.Add(new Tuple(weight, futureContract)); + spreadContract.legs.Add(new Tuple(weight, futureContract)); } - return spreadContract; } - public static ContractType DetermineFutureOrSpread(string contract) { - if (contract.Contains("-")) { - return ContractType.FutureSpread; - } else { - return ContractType.Future; - } - } + // public static ContractType DetermineFutureOrSpread(string contract) { + // if (contract.Contains("-")) { + // return ContractType.FutureSpread; + // } else { + // return ContractType.Future; + // } + // } #endregion public static ContractType get_type(string contract_txt) { - FutureContract fc = ParseFutureContract(contract_txt); + FutureTradableContract fc = parse_future_contract(contract_txt); if (fc != null) { return ContractType.Future; } - FutureSpreadContract sc = ParseFutureSpreadContract(contract_txt); + FutureSpreadContract sc = parse_future_spread_contract(contract_txt); if (sc != null) { return ContractType.FutureSpread; } @@ -133,11 +215,11 @@ namespace bts.SilverPine { } public static Tuple get_symbol_category(string symbol) { - ContractUtils.FutureContract fc = ParseFutureContract(symbol); + ContractUtils.FutureTradableContract fc = parse_future_contract(symbol); if (fc != null) { return Tuple.Create(ContractCategory.FutureString, fc.symbol); } - var sc = ParseFutureSpreadContract(symbol); + var sc = parse_future_spread_contract(symbol); if (sc != null) { return Tuple.Create(ContractCategory.FutureString, sc.symbol); } @@ -165,18 +247,6 @@ namespace bts.SilverPine { public enum Exchange { CME } - public class FutureContract { - public string symbol; - public DateTime expiration; - public Exchange exchange; - } - public class FutureSpreadContract { - public List> legs; - public string symbol { - get { - return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol; - } - } - } + } } diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs index e4d966929..8a37ba4bf 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs @@ -1,21 +1,11 @@ -namespace BCG { +namespace bts.SilverPine { using BCG.Utilities.General; using bts.api; using bts.api.domain; using bts.api.synthetic; using bts.md; - using bts.spark; - using bts.utils; - - using mbts; - using mbts.co2; - using mbts.config; - using mbts.spark; - using mbts.ui; - - using Spark.Core.Data; - using Spark.Core.Disposables; + using BCG; using Spark.Core.Reactive; using System; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/HistoricalBarsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/HistoricalBarsStream.cs new file mode 100644 index 000000000..8de13d5f0 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/HistoricalBarsStream.cs @@ -0,0 +1,241 @@ +namespace bts.SilverPine { + using BCG; + + using System.Runtime.CompilerServices; + + using ZstdSharp.Unsafe; + + public class DateTimeRange { + public readonly DateTime range_beg; + public readonly DateTime range_end; + public DateTimeRange(DateTime range_beg_, DateTime range_end_) { + range_beg = range_beg_; + range_end = range_end_; + } + public static DateTimeRange parse(string range_beg_, string range_end_) { + DateTime range_beg = String.IsNullOrEmpty(range_beg_) ? DateTime.MinValue : DateTime.Parse(range_beg_); + DateTime range_end = String.IsNullOrEmpty(range_end_) ? DateTime.MaxValue : DateTime.Parse(range_end_); + return new DateTimeRange(range_beg, range_end); + } + } + + public class FutureBarsLoader { + public static string get_db_table_name(BarSize bar_size, string symbol, bool is_continous) { + switch (bar_size) { + case BarSize.ONE_DAY: { + if (is_continous) { + return $"bars_future_{symbol}_continous_1day"; + } else + return $"bars_future_{symbol}_all_1day"; + } + case BarSize.ONE_MIN: { + if (is_continous) { + return $"bars_future_{symbol}_continous_1min"; + } else + return $"bars_future_{symbol}_all_1min"; + } + } + Log.assert(false); + return null; + } + } + public enum BarSize { + ONE_DAY = 0, + ONE_MIN + } + + [DataStreamUserRespresentation("FuturesHistorical1DayBars")] + public class FuturesHistorical1DayBarsStream : ViewStream { + + // this gets the raw bars for a symbol (perhaps make it into a raw bars stream?) + public FuturesHistorical1DayBarsStream(string symbol) : + this(symbol, DateTime.MinValue, DateTime.MaxValue) { } + + // creates the non raw stream (ie for tradable contract, or continous contract) + public FuturesHistorical1DayBarsStream(string future_contract, DateTime range_beg, DateTime range_end) + : this(new ContractUtils.FutureContract(future_contract), new DateTimeRange(range_beg, range_end)) { } + + // this creates a tradable contracts bars streams or continous contracts stream. if get_all_rows is true, + // it gets the raw table with all contracts for that symbol + public FuturesHistorical1DayBarsStream(ContractUtils.FutureContract future_contract, + DateTimeRange dt_range) { + DateTime beg = dt_range.range_beg;//.ToUniversalTime(), + DateTime end = dt_range.range_end;//.ToUniversalTime(); + beg = beg.Date; + end = end.Date.Add(new TimeSpan(23, 59, 59)); + // this returns rows from beg and end inclusive + Func get_sqlite_str = (table_name) => { + string range_beg_iso8601 = beg.ToString("yyyy-MM-ddTHH:mm:ss"), + range_end_iso8601 = end.ToString("yyyy-MM-ddTHH:mm:ss"); + string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'"; + return new SQLiteStream(SQLiteHelper.db_name, sql_str); + }; + string symbol = future_contract.symbol(); + if (future_contract.wildcard_contract != null) { + string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false); + ViewStream rs = get_sqlite_str(table_name); + //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime); + mySetRealSource(rs); + return; + } + if (future_contract.wildcard_contract != null) { + string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false); + ViewStream rs = get_sqlite_str(table_name); + //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime); + mySetRealSource(rs); + return; + } + if (future_contract.is_continous_contract) { + string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true); + ViewStream rs = get_sqlite_str(table_name); + //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime); + mySetRealSource(rs); + return; + } else { + string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false); + ViewStream rs = get_sqlite_str(table_name); + //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime); + //TODO: filter it + rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'"); + mySetRealSource(rs); + } + } + } + + [DataStreamUserRespresentation("FuturesHistorical1MinBars")] + public class FuturesHistorical1MinuteBarsStream : ViewStream { + // same as the above but for one min + // this gets the raw bars for a symbol + public FuturesHistorical1MinuteBarsStream(string symbol) : + this(symbol, DateTime.MinValue, DateTime.MaxValue) { } + + // creates the non raw stream (ie for tradable contract, or continous contract) + public FuturesHistorical1MinuteBarsStream(string future_contract, DateTime range_beg, DateTime range_end) + : this(new ContractUtils.FutureContract(future_contract), new DateTimeRange(range_beg, range_end)) { } + + // this creates a tradable contracts bars streams or continous contracts stream. if get_all_rows is true, + // it gets the raw table with all contracts for that symbol + public FuturesHistorical1MinuteBarsStream(ContractUtils.FutureContract future_contract, + DateTimeRange dt_range) { + Func get_sqlite_str = (table_name) => { + string range_beg_iso8601 = dt_range.range_beg.ToString("yyyy-MM-ddTHH:mm:ss"), + range_end_iso8601 = dt_range.range_end.ToString("yyyy-MM-ddTHH:mm:ss"); + //string sql_str = $"select * from {table_name} where DATE(ts_event) >= '{range_beg_iso8601}' AND DATE(ts_event) <= '{range_end_iso8601}'"; + string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'"; + return new SQLiteStream(SQLiteHelper.db_name, sql_str); + }; + string symbol = future_contract.symbol(); + if (future_contract.wildcard_contract != null) { + string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false); + ViewStream rs = get_sqlite_str(table_name); + mySetRealSource(rs); + return; + } + if (future_contract.is_continous_contract) { + string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true); + ViewStream rs = get_sqlite_str(table_name); + mySetRealSource(rs); + return; + } else { + string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false); + ViewStream rs = get_sqlite_str(table_name); + // TODO: filter + rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'"); + mySetRealSource(rs); + } + } + } + + [DataStreamUserRespresentation("FutureHistoricalBars")] + public class FutureHistoricalBarsStream : ViewStream { + public FutureHistoricalBarsStream(string fut_contract, string range_beg, string range_end) { + DateTime dt_range_beg = DateTime.Parse(range_beg); + DateTime dt_range_end = DateTime.Parse(range_end); + TimeSpan diff = dt_range_end - dt_range_beg; + BarSize bar_size = diff < TimeSpan.FromDays(1.0) ? BarSize.ONE_MIN : BarSize.ONE_DAY; + ViewStream rs = new FutureHistoricalBarsStream(fut_contract, bar_size.ToString(), dt_range_beg, dt_range_end); + mySetRealSource(rs); + } + public FutureHistoricalBarsStream(string fut_contract, string bar_size, + DateTime dt_range_beg, DateTime dt_range_end) + : + this(new ContractUtils.FutureContract(fut_contract), Enum.Parse(bar_size, true), + new DateTimeRange(dt_range_beg, dt_range_end)) { } + + public FutureHistoricalBarsStream(ContractUtils.FutureContract fut_contract, BarSize bar_size, + DateTimeRange dt_range) { + switch (bar_size) { + case BarSize.ONE_DAY: { + ViewStream rs = new FuturesHistorical1DayBarsStream(fut_contract, dt_range); + mySetRealSource(rs); + break; + } + case BarSize.ONE_MIN: { + ViewStream rs = new FuturesHistorical1MinuteBarsStream(fut_contract, dt_range); + mySetRealSource(rs); + break; + } + } + } + } + + [DataStreamUserRespresentation("FuturePriceDiff")] + public class FutureHistoricalBarsPriceDiffStream : ViewStream { + + public FutureHistoricalBarsPriceDiffStream(string future_contract, string dt1, string dt2) + : + this(new ContractUtils.FutureContract(future_contract), DateTime.Parse(dt1), DateTime.Parse(dt2), "close", ViewStreamDatabase.instance) { + } + + + public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2) + : + this(new ContractUtils.FutureContract(future_contract), dt1, dt2, "close", ViewStreamDatabase.instance) { + } + + public FutureHistoricalBarsPriceDiffStream(string future_contract, DateTime dt1, DateTime dt2, string col) + : + this(new ContractUtils.FutureContract(future_contract), dt1, dt2, col, ViewStreamDatabase.instance) { + } + + + public FutureHistoricalBarsPriceDiffStream(ContractUtils.FutureContract fc, DateTime dt1, DateTime dt2, string col, ViewStreamDatabase vsd) { + BarSize bsize = dt2 - dt1 >= TimeSpan.FromDays(1.0) ? BarSize.ONE_DAY : BarSize.ONE_MIN; + ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(dt1, dt2)); + + rs = new ColumnDifferenceStream(rs, r => { + DateTime r_dt = r.get_datetime("ts_event"); + var rv = dt1.Equals(r_dt); + return rv; + }, + //r => dt2.Equals(r.get_datetime("ts_event")), col, col); + r => { + DateTime r_dt = r.get_datetime("ts_event"); + var rv = dt2.Equals(r_dt); + return rv; + }, col, col); + rs = new NewProjectStream(rs, "diff !"); + mySetRealSource(rs); + } + } + + // takes the contract name and expands it into multuiple columns corresponding to the contracts. + // the contract columns are a union, of all properties with some of them empty + public class ContractExpanderStream : ViewStream { + private static IBlueTableSchema map_schema(IBlueTableSchema schema) { + return schema.add_column(TypeCode.String, "contract_type"); + } + public ContractExpanderStream(ViewStream parent_str, string contract_col_name) { + ViewStream rs = new MapStream(parent_str, (schema) => map_schema(schema), + r => { + string contract = r.get(contract_col_name); + var ct = ContractUtils.get_type(contract); + MutableBlueRow rv = new MutableBlueRow(map_schema(r.schema), r); + rv.setValue("contract_type", ct.ToString()); + return rv; + }); + mySetRealSource(rs); + } + } + +} diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/VolumeAggregationStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/VolumeAggregationStream.cs new file mode 100644 index 000000000..87d75d92c --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Bars/VolumeAggregationStream.cs @@ -0,0 +1,46 @@ +using BCG; + +namespace bts.SilverPine { + /* + ---------------------------------------------------------------------- + *Expression* + ---------------------------------------------------------------------- + //r=FutureHistoricalBars("ES*", "02/01/2024 09:30 AM", "02/01/2024 09:30 PM") + r=FutureHistoricalBars("ES*","One_Day", 02/01/2024, 02/01/2025) + r=contractExpander(r,"symbol") + r=filter(r,"contract_type='FutureSpread'") + r=aggregate(r,"symbol","volume:Sum","ts_event") + r=aggregate(r,"volume_sum:Max") + r=newProject(r,"volume_sum_max:volume symbol !") + + ---------------------------------------------------------------------- + *Schema* + ---------------------------------------------------------------------- + symbol System.String + id System.Int32 + volume System.Double + */ + + [DataStreamUserRespresentation("VolumeAgg")] + public class VolumeAggregationStream : ViewStream { + public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end) + : this(future_contract, contract_type, range_beg, range_end, ViewStreamDatabase.instance) { } + + public VolumeAggregationStream(string future_contract, string contract_type, string range_beg, string range_end) + : + this(future_contract, contract_type, DateTime.Parse(range_beg), DateTime.Parse(range_end), ViewStreamDatabase.instance) { } + + public VolumeAggregationStream(string future_contract, string contract_type, DateTime range_beg, DateTime range_end, ViewStreamDatabase db) { + future_contract = future_contract.EndsWith("*") ? future_contract : future_contract + "*"; + ViewStream r; + r = new FutureHistoricalBarsStream(future_contract, + range_beg.ToString(), range_end.ToString()); + r = new ContractExpanderStream(r, "symbol"); + r = new FilterStream(r, $"contract_type='{contract_type}'"); + r = new NonIncGroupByStream(r, "symbol", "volume:Sum", "ts_event"); + r = new NonIncGroupByStream(r, "volume_sum:Max"); + r = new NewProjectStream(r, "volume_sum_max:volume symbol !"); + mySetRealSource(r); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs index c4a7d1ec0..a365b4a72 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs @@ -1,68 +1,117 @@ +// namespace bts.SilverPine { +// using bts; +// using BCG; +// using MongoDB.Driver.Linq; +// using System.ComponentModel.DataAnnotations; +// using ZstdSharp.Unsafe; + +// [DataStreamUserRespresentation("VWAP")] +// public class VWAPStream : ViewStream { +// // returns the VWAP stream for each contract +// private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) { +// var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt); +// ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params); +// // stream is grouped per contract and then vwap is calculated per contract +// GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract", +// (rows, idx) => { +// Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}"); +// IBlueRow ir = rows.ElementAt((int)idx); +// var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt); +// var volumeprices = sel_rows.Select(r => Tuple.Create(r.get("volume"), r.get("close"))).ToList(); +// double vwap = IndicatorCalc.calculate_VWAP(volumeprices); +// IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap")); +// MutableBlueRow nr = new MutableBlueRow(schema, ir); +// //vwap = -313.987; +// nr.setValue("vwap", vwap); +// return nr; +// }); +// return rs; +// } +// // the args want dt1 > dt2, if its nto we, swap them +// static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) { +// if (dt1 < dt2) { +// DateTime aux = dt1; +// dt1 = dt2; +// dt2 = aux; +// } +// } + +// // this assumes the ohlc_str has diff contracts +// public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) { +// swap_if_necessary(ref begin_dt, ref end_dt); +// Tuple ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract); +// string symbol = String.Empty; +// symbol = ccat_sym_contr.Item2; +// var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt); +// ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt); +// // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract, +// // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it +// ViewStream rs = vwap_str; +// switch (ccat_sym_contr.Item1) { +// case ContractUtils.ContractCategory.FutureString: +// case ContractUtils.ContractCategory.SpreadString: +// rs = new FilterStream(vwap_str, r => r.get("contract").CompareTo(ccat_sym_contr.Item2) == 0); +// break; +// case ContractUtils.ContractCategory.Symbol: +// rs = new FuturesActiveContractStream(vwap_str); +// break; +// } +// rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0); +// rs = new NewProjectStream(rs, "! vwap"); +// mySetRealSource(rs); +// } + +// public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt) +// : +// this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { } +// } +// } + +using System; + +using BCG; + +using bts.api; + namespace bts.SilverPine { - using bts; - using BCG; - using MongoDB.Driver.Linq; - using System.ComponentModel.DataAnnotations; - using ZstdSharp.Unsafe; + /* + ---------------------------------------------------------------------- + *Expression* + ---------------------------------------------------------------------- + //nl("Find Settlement price of the SP500 future in the first week of 2023.") + + r=FutureHistoricalBars("ES", "One_Day", "1/1/2024", "2/1/2024") + r=math(r, "vprice = volume * close") + r=aggregate(r, "vprice:Sum, volume:Sum") + r=math(r, "vwap = vprice_sum / volume_sum") + r=newProject(r, "vwap !") + + ---------------------------------------------------------------------- + *Schema* + ---------------------------------------------------------------------- + id System.Int32 + vwap System.Double + */ [DataStreamUserRespresentation("VWAP")] public class VWAPStream : ViewStream { - // returns the VWAP stream for each contract - private static ViewStream build_vwap_stream(string symbol, DateTime begin_dt, DateTime end_dt) { - var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt); - ViewStream ohlc_str = new OHLCVStream(symbol, ohlcv_params); - // stream is grouped per contract and then vwap is calculated per contract - GlobalMapStream rs = new GlobalMapStream(ohlc_str, "ts_event", "contract", - (rows, idx) => { - Log.Gen.info($"VWAPStream.GlobalMapStream delegate n_rows: {rows.Count} idx: {idx}"); - IBlueRow ir = rows.ElementAt((int)idx); - var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt); - var volumeprices = sel_rows.Select(r => Tuple.Create(r.get("volume"), r.get("close"))).ToList(); - double vwap = IndicatorCalc.calculate_VWAP(volumeprices); - IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap")); - MutableBlueRow nr = new MutableBlueRow(schema, ir); - //vwap = -313.987; - nr.setValue("vwap", vwap); - return nr; - }); - return rs; - } - // the args want dt1 > dt2, if its nto we, swap them - static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) { - if (dt1 < dt2) { - DateTime aux = dt1; - dt1 = dt2; - dt2 = aux; - } - } - - // this assumes the ohlc_str has diff contracts - public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) { - swap_if_necessary(ref begin_dt, ref end_dt); - Tuple ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract); - string symbol = String.Empty; - symbol = ccat_sym_contr.Item2; - var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt); - ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt); - // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract, - // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it - ViewStream rs = vwap_str; - switch (ccat_sym_contr.Item1) { - case ContractUtils.ContractCategory.FutureString: - case ContractUtils.ContractCategory.SpreadString: - rs = new FilterStream(vwap_str, r => r.get("contract").CompareTo(ccat_sym_contr.Item2) == 0); - break; - case ContractUtils.ContractCategory.Symbol: - rs = new FuturesActiveContractStream(vwap_str); - break; - } - rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0); - rs = new NewProjectStream(rs, "! vwap"); - mySetRealSource(rs); - } - - public VWAPStream(string symbol_or_contract, string begin_dt, string end_dt) + public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt) : - this(symbol_or_contract, DateTime.Parse(begin_dt), DateTime.Parse(end_dt)) { } + this(symbol_or_contract, beg_dt, end_dt, ViewStreamDatabase.instance) { } + + public VWAPStream(string symbol_or_contract, DateTime beg_dt, DateTime end_dt, ViewStreamDatabase db) { + var fc = new ContractUtils.FutureContract(symbol_or_contract); + ViewStream r = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(beg_dt, end_dt)); + + r = new MathStream(r, + "vprice = volume * close"); + r = new NonIncGroupByStream(r, + "vprice:Sum, volume:Sum"); + r = new MathStream(r, + "vwap = vprice_sum / volume_sum"); + r = new NewProjectStream(r, + "vwap !"); + mySetRealSource(r); + } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs index 14ddb97ae..10027bb5b 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs @@ -75,4 +75,36 @@ namespace bts.SilverPine { mySetRealSource(rs); } } +} + + +namespace bts.SilverPine { + using BCG; + + using bts.api; + + using Newtonsoft.Json.Serialization; + + [DataStreamUserRespresentation("FuturesContinousContract")] + public class FuturesContinousContractStream : ViewStream { + // the argument is symbol, like "ES" etc and not the contract itself + public FuturesContinousContractStream(string symbol) : this(symbol, ViewStreamDatabase.instance) { } + public FuturesContinousContractStream(string symbol, ViewStreamDatabase db) { + ViewStream r = new FuturesHistorical1DayBarsStream(symbol); + // filtering for futures only + r = new FilterStream(r, r => { + string contract = r.get("symbol"); + var fc = new ContractUtils.FutureContract(contract); + return fc.future_tradable_contract != null; + }); + r = new NonIncGroupByStream(r, + "ts_event", + "volume:Max"); + r = new NewProjectStream(r, + "ts_event symbol !"); + r = new SortStream(r, + "-ts_event"); + mySetRealSource(r); + } + } } \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs index 7291a773b..afc1ba83b 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs @@ -2,6 +2,93 @@ namespace bts.SilverPine { using bts; using BCG; using MongoDB.Driver; + + // OHLCV request breakdown + // symbol_or_contract (if symbol, get continous, otherwise get raw and filter it) + // bar_size (one min, one day) + // live/historic + // time range + + // internal code oranization only + public class OHLCV_1min_stream : ViewStream { + + public OHLCV_1min_stream(string symbol_or_contract, string range_beg, string range_end) : + this(symbol_or_contract, DateTime.Parse(range_beg), DateTime.Parse(range_end)) { } + + public OHLCV_1min_stream(string symbol_or_contract, DateTime range_beg, DateTime range_end) { + Func get_sqlite_str = (symbol, is_continous_contracts) => { + string table_name = is_continous_contracts ? $"ohlcv_1min_{symbol}_continous" : $"{symbol}_ohlcv_1min"; + string db_str = $"Data Source={SQLiteHelper.location}{SQLiteHelper.db_name};Version=3"; + string range_beg_iso8601 = range_beg.ToString("yyyy-MM-ddTHH:mm:ss"), + range_end_iso8601 = range_end.ToString("yyyy-MM-ddTHH:mm:ss"); + string sql_str = $"select * from {table_name} where DATE(ts_event) >= '{range_beg_iso8601}' AND DATE(ts_event) <= '{range_end_iso8601}'"; + return new SQLiteStream(db_str, sql_str); + }; + + Tuple tup = ContractUtils.get_symbol_category(symbol_or_contract); + var contract_cat = tup.Item1; + switch (contract_cat) { + case ContractUtils.ContractCategory.Symbol: { + ViewStream res = get_sqlite_str(tup.Item2, true); + mySetRealSource(res); + // this creates the continous stream for 1 min bars, but we store it in the DB, so we just read it from it + // ViewStream ohlcv_1min = get_sqlite_str(tup.Item2, false); + // // the following is whats written to the DB in the _contionous table + // ohlcv_1min = new DateTimeExpanstionStream(ohlcv_1min, "ts_event"); + // ViewStream active_contr_str = new FuturesActiveContractStream(tup.Item2); + // ViewStream res = new NonIncJoinStream(ohlcv_1min, "ts_event_date symbol", "ts_event ts_event_date symbol open high low close volume", + // active_contr_str, "ts_event contract", "ts_event:date contract:continous_contract"); + // res = new NullFilterStream(res, "continous_contract", true); + // res = new NewProjectStream(res, "ts_event date continous_contract open high low close volume !"); + // mySetRealSource(res); + break; + } + default: { + ViewStream sql_str = get_sqlite_str(tup.Item2, false); + ViewStream rs = new FilterStream(sql_str, $"symbol='{symbol_or_contract}'"); + // filter here + mySetRealSource(rs); + break; + } + } + } + } + + // this is to make bigger bars form smaller bars. Right now the assumption it only works + // with symbols vs contracts, and the 1min <= aggregation_period < 1day + [DataStreamUserRespresentation("OHLCVAggregated")] + public class OHLCVAggregatedStream : ViewStream { + public OHLCVAggregatedStream(string symbol, string agregation_period, string range_beg, string range_end) : + this(symbol, TimeBasedFilter.parse_duration(agregation_period), DateTime.Parse(range_beg), DateTime.Parse(range_end)) { + } + + public OHLCVAggregatedStream(string symbol, TimeSpan aggregation_period, DateTime range_beg, DateTime range_end) { + Tuple tup = ContractUtils.get_symbol_category(symbol); + Log.assert(tup.Item1 == ContractUtils.ContractCategory.Symbol); + if (aggregation_period < TimeSpan.FromDays(1.0) && + aggregation_period > TimeSpan.FromMinutes(1.0)) { + ViewStream r = new OHLCV_1min_stream(symbol, range_beg, range_end); + r = new DateTimeExpanstionStream(r, + "ts_event"); + double tot_secs = aggregation_period.TotalSeconds; + r = new MathStream(r, + $"bucket = Floor(ts_event_seconds_of_day / {tot_secs})"); + r = new NonIncGroupByStream(r, + "bucket", + // column:operation + "open:first, high:max, low:min, close:last, volume:sum", + "ts_event"); + r = new SortStream(r, + "-ts_event"); + mySetRealSource(r); + } else { + // fix it later + Log.assert(false); + } + } + } + + [DataStreamUserRespresentation("OHLCV")] // This stream retreives the OHLCV table as is, and fires it as-is. // The only logic here is as follows: @@ -80,12 +167,13 @@ namespace bts.SilverPine { switch (ohlcv_params.type) { case OHLCVType.LIVE: case OHLCVType.LIVE_AND_HISTORIC: + // TODO Log.assert(false); break; case OHLCVType.HISTORIC: // TODO: assuming 1 day only for historic string table_name = get_table_from_barsize(ohlcv_params.bar_size, symbol);// $"{symbol}_OHLCV_1DAY"; - //string table_name = $"{symbol}_{ohlcv_params.bar_size}"; + //string table_name = $"{symbol}_{ohlcv_params.bar_size}"; string db_str = $"Data Source={_location}{_db_name};Version=3"; string sql_str = $"select * from {table_name}"; // "Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;" @@ -111,6 +199,11 @@ namespace bts.SilverPine { : this(symbol, Enum.Parse(bar_size)) { } + // aggregates the bars + public OHLCVStream(string symbol, TimeSpan agregation_period) { + + } + } diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs index 383a7becb..07adc7b59 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs @@ -3,49 +3,22 @@ namespace bts.SilverPine { using bts; - using DnsClient.Protocol; - - using MongoDB.Driver; - [DataStreamUserRespresentation("Settlement")] public class SettlementPriceStream : ViewStream { public SettlementPriceStream(string future_name, string date_txt) - : this(future_name, date_txt, String.Empty) { } + : this(future_name, date_txt, date_txt) { } public SettlementPriceStream(string future_name, string begin_dt, string end_dt) : this(future_name, DateTime.Parse(begin_dt).Date, DateTime.Parse(end_dt).Date) { } public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt) - : this(symbol_or_contract, begin_dt, DateTime.MinValue) { } + : this(symbol_or_contract, begin_dt, begin_dt) { } public SettlementPriceStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) { - // DateTime begin_dt = DateTime.Parse(begin_dt_txt); - // DateTime end_dt = String.IsNullOrEmpty(end_dt_txt) ? DateTime.MinValue : DateTime.Parse(end_dt_txt); - Tuple ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract); - ContractUtils.ContractCategory contr_cat = ccat_sym_contr.Item1; - string symbol = ccat_sym_contr.Item2; - ViewStream settl_str = new OHLCVStream(symbol); - // TODO: close != settlement, get settlement from somewhere - ViewStream rs = new NewProjectStream(settl_str, "close:settlement"); - switch (contr_cat) { - case ContractUtils.ContractCategory.FutureString: - case ContractUtils.ContractCategory.SpreadString: - rs = new FilterStream(settl_str, r => r.get("contract").CompareTo(symbol_or_contract) == 0); - break; - case ContractUtils.ContractCategory.Symbol: - rs = new FuturesActiveContractStream(settl_str); - break; - } - if (DateTime.MinValue.Equals(end_dt)) { - rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0); - } - rs = new FilterStream(rs, r => { - DateTime dt = r.get_datetime("ts_event"); - return dt <= begin_dt && dt >= end_dt; - }); - rs = new MathStream(rs, "settlement = close * 1.0"); - rs = new NewProjectStream(rs, " ! ts_event:date settlement"); + ContractUtils.FutureContract fc = new ContractUtils.FutureContract(symbol_or_contract); + ViewStream rs = new FutureHistoricalBarsStream(fc, BarSize.ONE_DAY, new DateTimeRange(begin_dt, end_dt)); + rs = new NewProjectStream(rs, "symbol ts_event:date close:settlement !"); mySetRealSource(rs); } diff --git a/src/DataStreamSuite/SharedStreams/NL/FewShotPrompts.json b/src/DataStreamSuite/SharedStreams/NL/FewShotPrompts.json index 96209e815..c0eb970de 100644 --- a/src/DataStreamSuite/SharedStreams/NL/FewShotPrompts.json +++ b/src/DataStreamSuite/SharedStreams/NL/FewShotPrompts.json @@ -41,70 +41,74 @@ }, { "category": "VWAP", - "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second argument is where the VWAP has to be calculated (also the beginning of the time window), and the third argument is datetime specifying the end of the time window. So, for example, a 20-day VWAP on 9/30/2023, the first argument will be 9/30/2023, and the third argument will be 9/10/2023 (20 days prior). This will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, etc. For the purpose of this few-shot prompts, let's assume today is 10/11/2024.", + "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, perio_end_date, period_start_date). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second and third argument define the end and start of the VWAP period, respectively. So for example, if we are calculating the 10 day VWAP on 10/15/2023, the period will be from 10/15/2023 going back to 10/05/2023. so the second argument will 10/05/2023 and the third argument will be 10/15/2023. The NL prompt will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, First day of quarter, Today etc. You will need to translate these as dates that need to be passed to the VWAP DSL function. ", "prompt": "What was the VWAP for SP 500 Future over the past 30 days?", "examples": [ { "natural_language": "Find the 20 day VWAP of the SP future on 04/22/2023.", - "dsl": "VWAP(\"ES\", 4/22/2023, 4/02/2023)" + "dsl": "VWAP(\"ES\", 4/02/2023, 4/22/2023)" }, { "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.", - "dsl": "VWAP(\"CL\", 10/10/2024, 10/1/2024)" + "dsl": "VWAP(\"CL\", 10/01/2024, 10/10/2024)" }, { "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.", - "dsl": "VWAP(\"ZN\", 10/11/2024, 9/11/2024)" + "dsl": "VWAP(\"ZN\", 9/11/2024, 10/11/2024)" }, { "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.", - "dsl": "VWAP(\"NG\", 9/27/2024 17:00:00, 9/27/2024 7:00:00)" + "dsl": "VWAP(\"NG\", 9/27/2024 7:00:00, 9/27/2024 17:00:00)" } ] }, { - "category": "Price Diff", - "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is OHLCVPriceDiff (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are dates, that the function finds the difference of prices at. Assume today is 10/15/2024", - "prompt": "What is the difference in SP 500 Future from July 10th to July 14th?", + "category": "FuturePriceDiff", + "description": "This is a few-shot learning example for an NL to DSL. The DSL function in question is FuturePriceDiff(symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500 and so on and so forth) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are datetimes (usually dates) that the function finds the difference of prices at. If the dates are given like \"first day of quarter\", \"first week of August\" I expect you to convert them into exact dates since the function only takes specific dates. The second argument should be the smaller datetime and the 3rd argument should be the end of the range, ie the higher one. Any prompt asking for price difference or price move should call FuturePriceDiff. If the dates you compute to pass to FuturePriceDiff are dates on which futures exchange might be closed (like 1/1, 12/25, Thanksgiving etc) please adjust the date to the nearest viable trading date, for example if the date falls on christmas day, change it to 12/24 and so on and so on and so forth, or the date happens to be a saturday or sunday, change it to Friday or Monday depending on which is closest", + "prompt": "What is the difference in SP 500 Future from July 10th to July 14th?", "examples": [ { "natural_language": "What is the difference in ZN price from July 10th to July 14th?", - "dsl": "OHLCVPriceDiff(\"ZN\", 7/14/2024, 7/10/2024)" + "dsl": "FuturePriceDiff(\"ZN\", 7/10/2024, 7/14/2024)" }, { "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?", - "dsl": "OHLCVPriceDiff(\"ES\", 9/10/2024, 9/5/2024)" + "dsl": "FuturePriceDiff(\"ES\", 9/5/2024, 9/10/2024)" }, { "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?", - "dsl": "OHLCVPriceDiff(\"NGZ4\", 8/30/2024, 8/20/2024)" + "dsl": "FuturePriceDiff(\"NGZ4\", 8/20/2024, 8/30/2024)" + }, + { + "natural_language": "What is the price move in SP 500 Future from Aug 30th 2023 to October 10th 2023", + "dsl": "FuturePriceDiff(\"ES\", 8/30/2023, 10/10/2023)" + }, + { + "natural_language": "What is the price move in SP 500 Future during teh first quarter of last year", + "dsl": "FuturePriceDiff(\"ES\", 1/1/2023, 3/31/2023)" } ] }, { - "category": "Absolute Volume", - "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is AbsVolume(symbol, ContractType, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc). The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment. For this prompt, assume today is 10/11/2024 ", + "category": "Aggregate Volume", + "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is VolumeAgg(symbol, contract_type, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc). The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment.", "prompt": "Which SP 500 Future traded the most between 7/14/2024 and 7/10/2024", "examples": [ { - "natural_language": "Which SP Future traded the most between 7/14/2024 and 7/10/2024", - "dsl": "AbsVolume(\"ES\", \"Future\", 7/14/2024, 7/10/2024)" + "natural_language": "Which SP Future traded the most between 7/10/2024 and 7/14/2024", + "dsl": "VolumeAgg(\"ES\", \"Future\", 1/1/2024, 1/7/2024)" }, { - "natural_language": "Which Gold Spread traded the most between Today and Sept 20th", - "dsl": "AbsVolume(\"GC\", \"FutureSpread\", 10/1/2024, 9/20/2024)" + "natural_language": "Which Gold Spread traded the most in the first month of this year", + "dsl": "VolumeAgg(\"GC\", \"FutureSpread\", 1/1/2024, 1/31/2024)" }, { - "natural_language": "Which 10 year Spread traded the highest volume last week?", - "dsl": "AbsVolume(\"ZN\", \"FutureSpread\", 9/27/2024, 9/23/2024)" + "natural_language": "Which 10 year Spread traded the highest volume last week to now?(in this example(only) we are assuming today is 11/26/2024)", + "dsl": "VolumeAgg(\"ZN\", \"FutureSpread\", 11/19/2024, 11/26/2024)" }, { - "natural_language": "Which Corn Spread traded the most today?", - "dsl": "AbsVolume(\"ZC\", \"FutureSpread\", 10/1/2024, 10/1/2024)" - }, - { - "natural_language": "Which Gold Future traded the most today?", - "dsl": "AbsVolume(\"GC\", \"Future\", 9/30/2024, 9/30/2024)" + "natural_language": "Which Corn Spread traded the most today (in this example(only) we are assuming today is 11/26/2024)?", + "dsl": "VolumeAgg(\"ZC\", \"FutureSpread\", 11/26/2024, 11/26/2024)" } ] }, @@ -218,6 +222,17 @@ "dsl": "r=standardDeviation(\"ZN\", \"30d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")" } ] + }, + { + "category": "Price Movement", + "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking for the average price movement for e-minis (aka ES aka SP Futures aka SP 500 Futures)) on Wednesday between 9:30 am and 9:40 AM? the multiline DSL is as follows (along with the explaination in comments before each line, dont include the comments in the asnwer): //following line gets the 1 min ohlcv bars of ES \n r=OHLCV_1min_stream(\"ES\")\n //this passes it to the filter stream, which filters the rows for Wednesday, and time bw 9:30 and 9:40.\nr=filter(r,\"ts_event_day = 'Wednesday' and ts_event_hour = 9 and ts_event_min >= 30 and ts_event_min < 40\")\n// this aggregates it using the date as the agregation key, it find the max of the column called high, and the low of the column called min, to find the max and the min prices in that time window\nr=aggregate(r, \"ts_event_date\", \"high: max, low: min\")\n/// this then passes the previous result to the math stream operator which calculates the move, which is the difference between high_max and low_min (columns that were added from the previous aggregate)\nr=math(r,\"move = high_max - low_min\")\n// At the end we compute the mean of the move column, notice this is the total mean, so we are not providing an aggregation col to agregate\nr=aggregate(r, \"move:mean\")\n// we project out every col by the ! operator and call rename move_mean as average_vol\nr=newProject(r,\"move_mean:average_move !\")", + "prompt": "What is the average price movement for SP Futures (ES) on Wednesday between 9:30 am and 9:40 AM ?", + "examples": [ + { + "natural_language": "What is the average price movement for SP Futures (ES) on Wednesday between 9:30 am and 9:40 AM?", + "dsl": "r=OHLCV_1min_stream(\"ES\")\nr=filter(r,\"ts_event_day = 'Wednesday' and ts_event_hour = 9 and ts_event_min >= 30 and ts_event_min < 40\")\nr=aggregate(r, \"ts_event_date\", \"high: max, low: min\")\nr=math(r,\"move = high_max - low_min\")\nr=aggregate(r, \"move:mean\")\nr=newProject(r,\"move_mean:average_move !\")" + } + ] } ] } \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/NL/NLStream.cs b/src/DataStreamSuite/SharedStreams/NL/NLStream.cs index 977641f01..9a714d91b 100644 --- a/src/DataStreamSuite/SharedStreams/NL/NLStream.cs +++ b/src/DataStreamSuite/SharedStreams/NL/NLStream.cs @@ -1,4 +1,4 @@ -namespace sbts.SilverPine { +namespace bts.SilverPine { using BCG; using bts; diff --git a/src/client/spark/Spark.csproj b/src/client/spark/Spark.csproj index 5ee8ed0dc..a1203d145 100644 --- a/src/client/spark/Spark.csproj +++ b/src/client/spark/Spark.csproj @@ -37,7 +37,7 @@ - + diff --git a/src/client/spark/SpreadBox/DateTimeFormatter.cs b/src/client/spark/SpreadBox/DateTimeFormatter.cs deleted file mode 100644 index e38f61bbd..000000000 --- a/src/client/spark/SpreadBox/DateTimeFormatter.cs +++ /dev/null @@ -1,41 +0,0 @@ -// using System; -// using System.Collections.Generic; -// using System.Linq; -// using System.Text; - -// namespace BCG -// { -// public class DateTimeFormatter -// { -// private delegate string MyFormatDateTime (DateTime v); -// public enum ExpirationShortForms -// { -// DayMonthYearDiff2000, -// MonthYearDiff2000 -// } -// public enum DateTimeFormats { Regular, ShortDate, DayMonthYearDiff2000, MonthYearDiff2000 } -// private static Dictionary _local_formats = null; -// private static void _init() -// { -// if (_local_formats == null) -// { -// _local_formats = new Dictionary(); -// _local_formats.Add(DateTimeFormats.Regular.ToString(), v => v.ToString()); -// _local_formats.Add(DateTimeFormats.ShortDate.ToString(), v => v.ToShortDateString()); -// _local_formats.Add(DateTimeFormats.DayMonthYearDiff2000.ToString(), -// (DateTime v) => { return String.Format("{0} {1}{2}", v.Day, BCG.Utilities.Dates.DateCode.getMonthSymbol(v.Month), v.Year - 2000); }); -// _local_formats.Add(DateTimeFormats.MonthYearDiff2000.ToString(), -// (DateTime v) => { return String.Format("{0}{1}", BCG.Utilities.Dates.DateCode.getMonthSymbol(v.Month), v.Year - 2000); }); -// } -// } -// public static string format(DateTime val, string format) -// { -// _init (); -// MyFormatDateTime f = null; -// if (_local_formats.TryGetValue(format, out f)) -// return f(val); -// else -// return val.ToString(format); -// } -// } -// } \ No newline at end of file