diff --git a/externals b/externals index 8069cc893..ff5f02bbd 160000 --- a/externals +++ b/externals @@ -1 +1 @@ -Subproject commit 8069cc893f5cef3b6cb0e989bbdf6514cc153d29 +Subproject commit ff5f02bbde0f584493835ac267af34b31e637845 diff --git a/src/DataStreamSuite/DataStream/DataStream.csproj b/src/DataStreamSuite/DataStream/DataStream.csproj index 8c8ed8d6b..b929774ae 100644 --- a/src/DataStreamSuite/DataStream/DataStream.csproj +++ b/src/DataStreamSuite/DataStream/DataStream.csproj @@ -18,8 +18,10 @@ + + diff --git a/src/DataStreamSuite/DataStream/Misc/DSLStream.cs b/src/DataStreamSuite/DataStream/Misc/DSLStream.cs index 5824a60e1..c652e15e8 100644 --- a/src/DataStreamSuite/DataStream/Misc/DSLStream.cs +++ b/src/DataStreamSuite/DataStream/Misc/DSLStream.cs @@ -13,6 +13,7 @@ namespace bts.SilverPine { Log.Gen.info($"DSLStream.cons dsl: {dsl}"); _dsl = dsl; ViewStream rs = build_stream(dsl); + rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\""); mySetRealSource(rs); } @@ -23,7 +24,7 @@ namespace bts.SilverPine { return rv; } catch (Exception ex) { - bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}"); + Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}"); return null; } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/NullFilterStream.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/NullFilterStream.cs new file mode 100644 index 000000000..62feaf799 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/NullFilterStream.cs @@ -0,0 +1,22 @@ +namespace bts.SilverPine { + using BCG; + + using System; + + public class NullFilterStream : ViewStream { + public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) { + ViewStream rs = new FilterStream(parent_str, + r => { + bool is_dbnull = r.is_null(col_name); + if (is_dbnull) { + return false; + } + if (r.schema[col_name].type_code() == TypeCode.Double) { + return !Double.IsNaN(r.get(col_name)); + } + return true; + }); + mySetRealSource(rs); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs index adea44bac..092152d22 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs @@ -9,6 +9,10 @@ using mbts; namespace BCG { public class NonIncJoinSpec { public ViewStream stream; + // join col names are the names on which the streams are joined (for equality), + // data col names are the names in the original stream + // projected col naes are their new names, ie if its x:l_x, then x is in the + // data_col_names and l_x in projected cols public string[] join_col_names, data_col_names, projected_col_names; public NonIncJoinSpec() { } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs index 07b9f6528..92232eb71 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs @@ -124,11 +124,12 @@ namespace BCG { IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows)); timer.Stop(); _n_total_fires++; - if (timer.ElapsedMilliseconds >= 250) { - _n_expensive_fires++; - string log_msg = $"Join and map time for {_book_keeping_data.Count} " - + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}"; - } + //if (timer.ElapsedMilliseconds >= 250) { + _n_expensive_fires++; + string log_msg = $"Join and map time for {_book_keeping_data.Count} " + + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}"; + Log.Gen.info(log_msg); + //} var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket); myFireEvent(dp_out); //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows)); diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/Proximity/ProximityJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/Proximity/ProximityJoinStream.cs new file mode 100644 index 000000000..aeb26028b --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/Proximity/ProximityJoinStream.cs @@ -0,0 +1,68 @@ +namespace bts { + using BCG; + using BCG.Utilities.General; + + using System; + using System.Diagnostics; + + using ZstdSharp.Unsafe; + + // functon that joins the 2 rows and returns the outputput row. The function + // can return a null which means the 2 rows dont need to be joined + using JoinFunc = System.Func; + public class ProximityJoinStream : ViewStream, IWorker { + private ViewStream _lvs, _rvs; + private JoinFunc _join_func; + private bool _dirty = false; + private Stopwatch _stopwatch = new Stopwatch(); + private ViewStreamDatabase _vsd = ViewStreamDatabase.instance; + public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) { + _lvs = lvs; + _rvs = rvs; + _join_func = join_func; + } + + protected override void myOnFirstSubscriber() { + _lvs.onChange += _handle_data; + _rvs.onChange += _handle_data; + _vsd.addWorker(this); + } + + protected override void myOnLastSubscriber() { + _lvs.onChange -= _handle_data; + _rvs.onChange -= _handle_data; + _vsd.removeWorker(this); + } + + private void _handle_data(object dp_obj) { + _dirty = true; + doWork(); + } + + public void doWork() { + Func all_data_revcd = () => { + return _lvs.rows != null && _rvs.rows != null; + }; + _stopwatch.Start(); + if (_dirty && all_data_revcd()) { + _dirty = false; + BlueTable rv = null; + foreach (IBlueRow lr in _lvs.rows) { + foreach (IBlueRow rr in _rvs.rows) { + IBlueRow resr = _join_func(lr, rr); + if (resr == null) + continue; + rv = rv ?? new BlueTable(resr.schema); + rv.addRow(resr); + } + } + _stopwatch.Stop(); + Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs"); + var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket); + myFireEvent(dp); + } + + } + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/Proximity/TimeProximityJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/Proximity/TimeProximityJoinStream.cs new file mode 100644 index 000000000..0f48e5265 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Join/Proximity/TimeProximityJoinStream.cs @@ -0,0 +1,47 @@ +namespace bts.SilverPine { + using BCG; + + using System; + using System.CodeDom; + using System.Configuration; + using System.Reactive.Disposables; + + public class TimeProximityJoinStream : ViewStream { + private int _id = 0; + + public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col, + string pre_join_period, string post_join_period) : + this(ls, l_time_col, rs, r_time_col, TimeBasedFilter.parse_duration(pre_join_period), + TimeBasedFilter.parse_duration(post_join_period)) { } + + + public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col, + TimeSpan pre_join_period, TimeSpan post_join_period) { + Func create_row = (lr, rr) => { + var id = 0; + lock (this) { + id = _id++; + } + IBlueRow rv = BlueDataUtility.create_composite_row(lr, rr, id); + return rv; + }; + Func join_func = (lr, rr) => { + DateTime ldt = lr.get_datetime(l_time_col); + DateTime rdt = rr.get_datetime(r_time_col); + TimeSpan diff = ldt - rdt; + if (ldt < rdt) { + if (diff.Duration() > pre_join_period) { + return null; + } + } else { + if (diff > post_join_period) + return null; + } + return create_row(lr, rr); + }; + + ViewStream rstr = new ProximityJoinStream(ls, rs, join_func); + mySetRealSource(rstr); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs b/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs index 9fe3d6198..05766e90c 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs @@ -26,7 +26,7 @@ namespace BCG { return rv; } - //literal("table1", "x:123 y:1.001 z:abc;x:123 y:1.001 z:abc") + //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc") public LiteralStream(string table_name, string rows) { Func get_row = (json_row, id) => { Dictionary map = new Dictionary(); diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs b/src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs index 3b4e81539..098350ddf 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs @@ -137,10 +137,24 @@ namespace bts { HashSet removed_cols = new HashSet(); Dictionary mapped_to_orig_map = new Dictionary(); Dictionary orig_to_mapped = new Dictionary(); + // by default every col will show up, so if we have the projection as (a:a1) + // it will convert a to a1, + // examples: if a schema has cols a b c + // project(parent_str, "a:aa ") -> aa b c + // project(parent_str, "a:aa !c") -> aa b + // project(parent_str, "a:aa !") -> aa + // project(parent_str, "a:aa b = (aa*aa)!") -> aa b + bool hide_all = false; foreach (string p_tok in project_toks) { if (p_tok.StartsWith("!")) { - string removed_col = p_tok.Substring(1); - removed_cols.Add(removed_col); + if (p_tok.Length == 1) { + // if it is a "blanket "!"=> hide all cols except the one that + // are stated explicitly + hide_all = true; + } else { + string removed_col = p_tok.Substring(1); + removed_cols.Add(removed_col); + } } else if (p_tok.Contains(":")) { string[] map_toks = p_tok.Split(":"); string orig_col = map_toks[0]; @@ -155,12 +169,11 @@ namespace bts { } } } + // make the new schema List new_cols = new List(); foreach (ColumnInfo ci in input_schema.columnInfo) { if (ci.colName == "id") { - // if there's a hidden col, other columns will show by default, but if cols are being - // projected epxlicitely, than others wil be hidden ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType); new_cols.Add(nc); continue; @@ -172,12 +185,13 @@ namespace bts { ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType); new_cols.Add(nc); } else { - if (ci.colName == "id" || removed_cols.Count > 0) { - // if there's a hidden col, other columns will show by default, but if cols are being - // projected epxlicitely, than others wil be hidden - ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType); - new_cols.Add(nc); - } + if (hide_all) + continue; + //if (ci.colName == "id" || removed_cols.Count > 0) { + // unless columns are explicitely hidden, all columns show up + ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType); + new_cols.Add(nc); + //} } } IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray()); diff --git a/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs index 407ee4c24..818f3e60c 100644 --- a/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs +++ b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs @@ -1,10 +1,167 @@ using System; using System.Data; using System.Data.SQLite; +using System.Diagnostics; +using System.Collections; + using BCG; +using System.Linq; +using System.Data.Entity; +using System.Collections.Generic; + +namespace bts.SilverPine { + + public class DataRowWrapperRow : IBlueRow { + private readonly DataRow _row; + private readonly IBlueTableSchema _schema; + private readonly int _id; + public DataRowWrapperRow(IBlueTableSchema schema, DataRow row, int id) { + _row = row; + _id = id; + _schema = schema; + } + public virtual IBlueTableSchema schema { get { return _schema; } } + + public virtual object this[string col_name] { + get { + ColumnInfo ci = _schema[col_name]; + return this[ci.colNo]; + } + } + public virtual object this[int index] { + get { + if (_schema[index].colName == "id") + return _id; + return _row[index]; + } + } + + public virtual int Count { get { return _row.ItemArray.Length; } } + + public virtual bool IsSynchronized { get { return false; } } + + public virtual object SyncRoot { get { return this; } } + + public virtual void CopyTo(Array array, int index) { _row.ItemArray.CopyTo(array, index); } + + public virtual bool IsFixedSize { get { return true; } } + + virtual public bool IsReadOnly { get { return true; } } + + /// + /// The key can either be the string corrsponding to the column name or + /// the order of the column + /// + public virtual object this[object key] { + get { + if (key is string) + return this[(string)key]; + else if (key is int) + return this[(int)key]; + throw new BlueDataException("Wrong type while indexing the row"); + } + set { throw new NotSupportedException("The IBlueRow is readonly"); } + } + + public virtual ICollection Keys { get { return _schema.columnNames; } } + + public virtual ICollection Values { get { return _row.ItemArray; } } + + public virtual void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); } + + public virtual void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); } + + public virtual bool Contains(object key) { return _schema[(string)key] != null; } + + public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } + + IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); } + + public virtual void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); } + + } + + public class DataTableWrapperTable : IBlueRowCollection { + + public class DataTableWrapperTableEnumerator : IEnumerator { + private IEnumerator _enumerator; + private IBlueTableSchema _schema; + private int _id = 0; + public DataTableWrapperTableEnumerator(DataTable table) { + _enumerator = table.Rows.GetEnumerator(); + _schema = BlueDataUtility.getSchema(table); + + } + public object Current { + get { + DataRow r = _enumerator.Current as DataRow; + return new DataRowWrapperRow(_schema, r, _id); + } + } + public bool MoveNext() { + _id++; + return _enumerator.MoveNext(); + } + public void Reset() { + _id = 0; + _enumerator.Reset(); + } + } + + private readonly DataTable _table; + private IBlueTableSchema _schema; + private IEnumerator _enumerator; + + public DataTableWrapperTable(DataTable table) { + _table = table; + _schema = BlueDataUtility.getSchema(_table); + _enumerator = new DataTableWrapperTableEnumerator(table); + } + public virtual IBlueRow this[int id] { + get { + // its assumed the rows index == id + DataRow dr = _table.Rows[id]; + return new DataRowWrapperRow(_schema, dr, id); + } + set { + + } + } + + public virtual IBlueTableSchema schema { get { return _schema; } } + + public virtual bool containsID(int id) { return id < _table.Rows.Count; } + + // private List _rows() { + // List rv = new List(); + // int id = 0; + // foreach (DataRow r in _table.Rows) { + // rv.Add(new DataRowWrapperRow(r, id++)); + // } + // return rv; + // } + + public virtual IEnumerator GetEnumerator() { + // var rows = _rows(); + // return rows.GetEnumerator(); + return _enumerator; + } + + public virtual bool IsSynchronized { get { return false; } } + + public virtual object SyncRoot { get { return this; } } + + public virtual void CopyTo(Array array, int index) { + Log.assert(false); + //_rows().CopyTo(array as IBlueRow[], index); + } + + public virtual int Count { + get { return _table.Rows.Count; } + } + } -namespace bts { public static class SQLiteHelper { public static DataTable ExecuteSelectQuery(string connection_string, string query) { // Create a new DataTable to store the results @@ -35,6 +192,7 @@ namespace bts { [DataStreamUserRespresentation("SQLite")] public class SQLiteStream : ViewStream { private readonly string _connection_string, _query; + private Stopwatch _stopwatch = new Stopwatch(); public SQLiteStream(string connection_string, string query) { _connection_string = connection_string; _query = query; @@ -42,9 +200,15 @@ namespace bts { private void _read_from_sqlite_nd_fire() { + _stopwatch.Start(); DataTable dt = SQLiteHelper.ExecuteSelectQuery(_connection_string, _query); + _stopwatch.Stop(); + Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to read db: {_stopwatch.ElapsedMilliseconds} msecs"); + _stopwatch.Start(); IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt); - + //IBlueRowCollection rows = new DataTableWrapperTable(dt); + _stopwatch.Stop(); + Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs"); DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket); myFireEvent(dp); } @@ -63,3 +227,4 @@ namespace bts { } } + diff --git a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs index 5991a25e0..41eb76390 100644 --- a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs @@ -3,12 +3,15 @@ using System.Collections.Generic; using System.Text; using System.IO; using BCG.Utilities; +using System.Diagnostics; +using Microsoft.Data.Analysis; +using Parquet; +using Parquet.Data; -namespace BCG -{ + +namespace BCG { [DataStreamUserRespresentation("CSVSaver")] - public class CSVSaverStream : ViewStream, ISideEffectStream - { + public class CSVSaverStream : ViewStream, ISideEffectStream { private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)), new ColumnInfo("fileName", 1, typeof(string)), new ColumnInfo("nWrites", 2, typeof(int)), @@ -20,11 +23,9 @@ namespace BCG private int myNWrites = 0, myNMaxWrites = Int32.MaxValue; public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName) - : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) - { } + : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) { } - public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) - { + public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) { myStream = vs; myFullyQualifiedFileName = fullyQualifiedFileName; myOverwrite = overwrite; @@ -36,18 +37,14 @@ namespace BCG protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); } - private void myHandleChanges(object newValue) - { + private void myHandleChanges(object newValue) { IBlueRowCollection totalState = (newValue as DataPacket).newRows; - if (totalState.Count > 0 && myNWrites < myNMaxWrites) - { + if (totalState.Count > 0 && myNWrites < myNMaxWrites) { myNWrites++; int i = 0; StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write)); - foreach (IBlueRow r in totalState) - { - if (i++ == 0) - { + foreach (IBlueRow r in totalState) { + if (i++ == 0) { string header = String.Empty; foreach (string col in r.schema.columnNames) if (col != "id") @@ -59,20 +56,17 @@ namespace BCG writer.Flush(); } writer.Close(); - if (myFireResultSnapshot) - { + if (myFireResultSnapshot) { IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count); BlueTable nRows = new BlueTable(myResultSchema); nRows.addRow(nr); myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows))); - } - else + } else myFireEvent(newValue); } } - private string myGetNewFileName() - { + private string myGetNewFileName() { string[] tags = myFullyQualifiedFileName.Split('.'); if (myOverwrite) return tags[0] + ".csv"; @@ -80,14 +74,11 @@ namespace BCG return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv"; } - private static string myGetCSVRow(IBlueRow r) - { + private static string myGetCSVRow(IBlueRow r) { string rv = String.Empty; int i = 0; - foreach (string col in r.schema.columnNames) - { - if (col != "id") - { + foreach (string col in r.schema.columnNames) { + if (col != "id") { string colText = String.Empty; if (r[col] != null && r[col] != DBNull.Value) colText = r[col].ToString(); @@ -99,8 +90,7 @@ namespace BCG return rv; } - private string myFormateDateTime(DateTime dt) - { + private string myFormateDateTime(DateTime dt) { return string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}", dt.Year.ToString("0000"), @@ -114,52 +104,49 @@ namespace BCG } [DataStreamUserRespresentation("CSVReader")] - public class CSVReaderStream : ViewStream - { - private IBlueRowCollection myRows; - - public CSVReaderStream(string fileAddress) - { - myRows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(fileAddress); + public class CSVReaderStream : ViewStream { + private readonly string _file_address; + Stopwatch _stopwatch = new Stopwatch(); + public CSVReaderStream(string fileAddress) { + _file_address = fileAddress; } - protected override void myOnFirstSubscriber() - { - myFireEvent(new DataPacket(myRows, myRows, new BlueTable(myRows.schema))); + protected override void myOnFirstSubscriber() { + _stopwatch.Start(); + DataFrame df = DataFrame.LoadCsv(_file_address); + _stopwatch.Stop(); + bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs"); + _stopwatch.Start(); + IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address); + _stopwatch.Stop(); + bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs"); + DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket); + myFireEvent(dp_out); } } - [DataStreamUserRespresentation ("CSVReaderStreamMapper")] - public class CSVReaderStreamMapperStream : ViewStream - { - public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) - { + [DataStreamUserRespresentation("CSVReaderStreamMapper")] + public class CSVReaderStreamMapperStream : ViewStream { + public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) { mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema))); } } - public class CSVReaderStreamMapper : IStreamMapper - { + public class CSVReaderStreamMapper : IStreamMapper { private IBlueTableSchema mySchema; private readonly string[] myOutputSchemaStringArray; - public CSVReaderStreamMapper(string outputSchema) - { + public CSVReaderStreamMapper(string outputSchema) { myOutputSchemaStringArray = outputSchema.Split(' '); } - public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) - { - if (mySchema == null) - { + public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) { + if (mySchema == null) { ColumnInfo[] ci = inputSchema.columnInfo; - foreach (string col in myOutputSchemaStringArray) - { + foreach (string col in myOutputSchemaStringArray) { string colName = col.Substring(0, col.IndexOf('\\')).ToLower(); - for (int i = 0; i < ci.Length; i++) - { - if (ci[i].colName.ToLower().Equals(colName)) - { + for (int i = 0; i < ci.Length; i++) { + if (ci[i].colName.ToLower().Equals(colName)) { Type t = myGetType(col.Substring(col.IndexOf('\\') + 1)); ci[i] = new ColumnInfo(ci[i].colName, i, t); break; @@ -171,74 +158,52 @@ namespace BCG return mySchema; } - public IBlueRow map(IBlueRow row) - { + public IBlueRow map(IBlueRow row) { object[] vals = new object[row.schema.columnInfo.Length]; row.Values.CopyTo(vals, 0); - for (int i = 0; i < mySchema.columnInfo.Length; i++) - { + for (int i = 0; i < mySchema.columnInfo.Length; i++) { Type t = mySchema.columnInfo[i].colType; - try - { + try { vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture); - } - catch - { - if (t.Equals(typeof(Int16))) - { + } catch { + if (t.Equals(typeof(Int16))) { Int16 result; - if (Int16.TryParse((string)vals[i], out result)) - { + if (Int16.TryParse((string)vals[i], out result)) { vals[i] = result; continue; } double r2; - if (Double.TryParse((string)vals[i], out r2)) - { - try - { + if (Double.TryParse((string)vals[i], out r2)) { + try { vals[i] = (Int16)r2; continue; - } - catch { } + } catch { } } - } - else if (t.Equals(typeof(Int32))) - { + } else if (t.Equals(typeof(Int32))) { Int32 result; - if (Int32.TryParse((string)vals[i], out result)) - { + if (Int32.TryParse((string)vals[i], out result)) { vals[i] = result; continue; } double r2; - if (Double.TryParse((string)vals[i], out r2)) - { - try - { + if (Double.TryParse((string)vals[i], out r2)) { + try { vals[i] = (Int32)r2; continue; - } - catch { } + } catch { } } - } - else if (t.Equals(typeof(Int64))) - { + } else if (t.Equals(typeof(Int64))) { Int64 result; - if (Int64.TryParse((string)vals[i], out result)) - { + if (Int64.TryParse((string)vals[i], out result)) { vals[i] = result; continue; } double r2; - if (Double.TryParse((string)vals[i], out r2)) - { - try - { + if (Double.TryParse((string)vals[i], out r2)) { + try { vals[i] = (Int64)r2; continue; - } - catch { } + } catch { } } } vals[i] = DBNull.Value; @@ -247,8 +212,7 @@ namespace BCG return new BlueRow(mySchema, vals); } - private Type myGetType(string type) - { + private Type myGetType(string type) { type = type.ToLower(); if (type.Contains("int")) return typeof(int); diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs index d967c1409..ef9f2499f 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs @@ -16,11 +16,73 @@ using BCG.Extensions; using System.Text.Json; using System.Text.Json.Nodes; using Newtonsoft.Json; +using BCG; +using System.Runtime.CompilerServices; +namespace bts { + + // public class CompositeRow : IBlueRow { + // private IBlueRow _lrow, _rrow; + // private BlueRow _out_row; + // public CompositeRow(IBlueRow lrow, IBlueRow rrow) { + // _lrow = lrow; + // _rrow = rrow; + // IBlueTableSchema schema = _lrow.schema; + // List vals = new List(); + // foreach (var o in _lrow) { + // vals.Add(o); + // } + // foreach (ColumnInfo ci in _rrow.schema.columnInfo) { + // if (ci.colName == "id") { + // continue; + // } + // vals.Add(_rrow[ci.colNo]); + // schema = schema.add_column(ci.type_code(), ci.colName); + // } + // _out_row = new BlueRow(schema, vals.ToArray()); + // } + // public IBlueTableSchema schema { + // get { return _out_row.schema; } + // } + // public object? this[int index] { + // get { return _out_row[index]; } + // } + // public object? this[string colName] { + // get { + // return _out_row[colName]; + // } + // } + + // } +} namespace BCG { public static class BlueDataUtility { + public static IBlueRow create_composite_row(IBlueRow lr, IBlueRow rr, int id) { + IBlueTableSchema schema = lr.schema; + List vals = new List(); + foreach (ColumnInfo ci in lr.schema.columnInfo) { + if (ci.colName == "id") { + vals.Add(id); + } + vals.Add(lr[ci.colName]); + } + foreach (ColumnInfo ci in rr.schema.columnInfo) { + if (ci.colName == "id") { + continue; + } + schema = schema.add_column(ci.type_code(), ci.colName); + vals.Add(rr[ci.colName]); + } + IBlueRow r = new BlueRow(schema, vals.ToArray()); + return r; + } #region Extensions + public static bool is_null(this IBlueRow r, string col_name) { + object col_val = r[col_name]; + return col_val == null || DBNull.Value.Equals(col_val); + } + public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) { return BlueDataUtility.get_row_datetime_value(r, col_name, err_value); } diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs index 6c1f58e3c..019e8de7a 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs @@ -169,8 +169,10 @@ namespace BCG } #endregion + private Stopwatch _stopwatch = new Stopwatch(); private void myTryExpression(string expStr) { + _stopwatch.Restart(); myFreezeView(false); // force unfreeze the view DataStreamExpression oldExp = myExp; try @@ -344,6 +346,7 @@ namespace BCG // this is the stream callback private void myCallback(object o) { + _stopwatch.Stop(); var timer = Stopwatch.StartNew(); DataPacket dp = o as DataPacket; myCrono.Stop(); @@ -359,7 +362,8 @@ namespace BCG myToolStripStatusLabelRows.BackColor = Color.Black; myToolStripStatusLabelTime.ForeColor = Color.White; myToolStripStatusLabelRows.ForeColor = Color.White; - myToolStripStatusLabelTime.Text = "Last update: " + DateTime.Now.ToString("hh:mm:ss:fff tt"); + string time_text = "Last update: " + DateTime.Now.ToString("hh:mm:ss:fff tt") + $" time: {_stopwatch.ElapsedMilliseconds} msecs\t\t"; + myToolStripStatusLabelTime.Text = time_text; myToolStripStatusLabelRows.Text = "New = " + dp.newRows.Count + " Added = " + dp.addedRows.Count + " Deleted = " + dp.deletedRows.Count; Log.Ui.diag(Text); int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue; diff --git a/src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs b/src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs index 658a67b18..22f35d514 100644 --- a/src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs +++ b/src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs @@ -122,8 +122,9 @@ namespace bts.SilverPine { if (sc != null) { return ContractType.FutureSpread; } - Log.assert(false); - return bts.SilverPine.ContractUtils.ContractType.Spot; + // tehre are some non calendar spreads we cant parse + //Log.assert(false); + return bts.SilverPine.ContractUtils.ContractType.FutureSpread; } public static ContractCategory get_category(string symbol) { diff --git a/src/DataStreamSuite/SharedStreams/Business/EventsPriceActionStream.cs b/src/DataStreamSuite/SharedStreams/Business/EventsPriceActionStream.cs new file mode 100644 index 000000000..cf61fb4eb --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/EventsPriceActionStream.cs @@ -0,0 +1,23 @@ +namespace bts.SilverPine { + using BCG; + + // Finds the regions around specified events + [DataStreamUserRespresentation("EventsPriceAction")] + public class EventsPriceActionStream : ViewStream { + public EventsPriceActionStream(string symbol, string event_type, string pre_event_period, + string post_event_period, int n_events) : + this(symbol, event_type, TimeBasedFilter.parse_duration(pre_event_period), TimeBasedFilter.parse_duration(post_event_period), + n_events, ViewStreamDatabase.instance) { } + + public EventsPriceActionStream(string symbol, string event_type, TimeSpan pre_event_period, + TimeSpan post_event_period, int n_events, ViewStreamDatabase vsd) { + // TODO: figure out which bars to use, 1 min or 1 day or live, based on pre_event_period / post_event_period + FuturesActiveContractStream olhcv_str = new FuturesActiveContractStream(symbol); + ViewStream events_str = new EventsStream(vsd, event_type); + // todo + //events_str = new TopStream(events_str, n_events, "-ts"); + ViewStream rs = new TimeProximityJoinStream(olhcv_str, "ts_event", events_str, "ts", pre_event_period, post_event_period); + mySetRealSource(rs); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/ChartsUtils.cs b/src/DataStreamSuite/SharedStreams/Business/Indicators/ChartsUtils.cs similarity index 100% rename from src/DataStreamSuite/SharedStreams/Business/ChartsUtils.cs rename to src/DataStreamSuite/SharedStreams/Business/Indicators/ChartsUtils.cs diff --git a/src/DataStreamSuite/SharedStreams/Business/Indicators/IndicatorStream.cs b/src/DataStreamSuite/SharedStreams/Business/Indicators/IndicatorStream.cs new file mode 100644 index 000000000..f6aac792f --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/Indicators/IndicatorStream.cs @@ -0,0 +1,20 @@ +namespace bts.SilverPine { + using BCG; + + public class IndicatorStream : ViewStream { + enum Indicator { + StandardDeviation = 0 + } + public class IndicatorInfo { + Indicator indicator; + IDictionary parameters; + } + public IndicatorStream(string symbol_name, string indicator_name, string indicator_params_json) { + + } + public IndicatorStream(string symbol_name, IndicatorInfo info) { + + } + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/AbsoluteVolumeStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/AbsoluteVolumeStream.cs index d5d5e8d2d..5bf3e32bf 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/AbsoluteVolumeStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/AbsoluteVolumeStream.cs @@ -34,12 +34,22 @@ namespace bts.SilverPine { : this(symbol, Enum.Parse(ctype), d1, d2) { } + static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) { + if (dt1 < dt2) { + DateTime aux = dt1; + dt1 = dt2; + dt2 = aux; + } + } + private AbsoluteVolumeStream(string symbol, ContractUtils.ContractType ctype, DateTime d1, DateTime d2) { + swap_if_necessary(ref d1, ref d2); ViewStream ohlcv_str = _get_ohlcv_steam(symbol, d1, d2); ohlcv_str = new FilterStream(ohlcv_str, r => r.get_datetime("ts_event") <= d1 && r.get_datetime("ts_event") >= d2); ohlcv_str = new FilterStream(ohlcv_str, r => String.Compare(ctype.ToString(), r.get("contract_type"), true) == 0); ViewStream rs = new MaxStream(ohlcv_str, "volume"); + rs = new NewProjectStream(rs, "volume:total_volume contract !"); mySetRealSource(rs); } diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/ChartingHelper.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/ChartingHelper.cs index 34be327d0..a9d0e1f01 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/ChartingHelper.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/ChartingHelper.cs @@ -5,7 +5,45 @@ using System.Linq; using bts; namespace bts.SilverPine { - public static class ChartingHelper { + public static class IndicatorCalc { + public static double calculate_pearson_correlation(List prices1, List prices2) { + if (prices1 == null) + throw new ArgumentNullException(nameof(prices1), "First price list cannot be null."); + if (prices2 == null) + throw new ArgumentNullException(nameof(prices2), "Second price list cannot be null."); + if (prices1.Count != prices2.Count) + throw new ArgumentException("Price lists must have the same number of elements."); + if (prices1.Count < 2) + throw new ArgumentException("At least two data points are required to calculate correlation."); + + int n = prices1.Count; + double sumX = 0; + double sumY = 0; + double sumX2 = 0; + double sumY2 = 0; + double sumXY = 0; + + for (int i = 0; i < n; i++) { + double x = prices1[i]; + double y = prices2[i]; + + sumX += x; + sumY += y; + sumX2 += x * x; + sumY2 += y * y; + sumXY += x * y; + } + + double numerator = (n * sumXY) - (sumX * sumY); + double denominatorPart1 = (n * sumX2) - (sumX * sumX); + double denominatorPart2 = (n * sumY2) - (sumY * sumY); + double denominator = Math.Sqrt(denominatorPart1 * denominatorPart2); + + if (denominator == 0) + throw new InvalidOperationException("Denominator is zero, correlation is undefined."); + + return numerator / denominator; + } public static double calculate_VWAP(List> volumePriceData) { Log.Gen.info($"ChartingHelper.calculate_VWAP n_vals: {volumePriceData.Count}"); double totalVolume = 0; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RSIStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RSIStream.cs index e88fc4111..9aa58e72d 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RSIStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/RSIStream.cs @@ -14,7 +14,7 @@ namespace bts.SilverPine { (rows, idx) => { IBlueRow ir = rows.ElementAt((int)idx); IEnumerable closes = BlueDataUtility.get_colvalues(rows, "close", (int)idx + 1, period + 2); - double rsi = ChartingHelper.calculate_RSI(closes.ToList(), period); + double rsi = IndicatorCalc.calculate_RSI(closes.ToList(), period); IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "rsi")); MutableBlueRow nr = new MutableBlueRow(schema, ir); nr.setValue("rsi", rsi); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SMAStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SMAStream.cs index 996c7a226..e4939a2b2 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SMAStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/SMAStream.cs @@ -10,7 +10,7 @@ namespace bts.SilverPine { IBlueRow ir = rows.ElementAt((int)idx); IEnumerable closes = BlueDataUtility.get_colvalues(rows, "close", (int)idx + 1, period + 2); - double rsi = ChartingHelper.calculate_RSI(closes.ToList(), period); + double rsi = IndicatorCalc.calculate_RSI(closes.ToList(), period); IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "rsi")); MutableBlueRow nr = new MutableBlueRow(schema, ir); nr.setValue("rsi", rsi); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/StandardDeviationStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/StandardDeviationStream.cs new file mode 100644 index 000000000..5fff6091f --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/StandardDeviationStream.cs @@ -0,0 +1,40 @@ +namespace bts.SilverPine { + using BCG; + + using MathNet.Numerics.Statistics; + + // calculates the reazlied standard deviation of a series of prices + public class StandardDeviationStream : ViewStream { + public StandardDeviationStream(string symbol, string std_dev_period) + : + this(symbol, TimeBasedFilter.parse_duration(std_dev_period)) { } + + public StandardDeviationStream(string symbol, TimeSpan std_dev_period) { + // since this is a 1_day bar str, 90 rows => 90 days, so no need to filter + int n_days = std_dev_period.Days; + ViewStream ohlcv_str = new FuturesActiveContractStream(symbol); + IBlueTableSchema out_schema = null; + int id = 0; + Func, uint, IBlueRow> find_std_dev = (rows, idx) => { + int iidx = Convert.ToInt32(idx); + if (rows == null || rows.Count == 0) + return null; + IBlueRow ir = rows.ElementAt(iidx); + out_schema = out_schema ?? ir.schema.add_column(TypeCode.Double, "std_dev"); + MutableBlueRow rv = new MutableBlueRow(out_schema, ir); + rv.setValue("std_dev", Double.NaN); + IEnumerable last_n_close_prices = rows.Skip(iidx).Take(n_days).Select(r => r.get("close")); + if (last_n_close_prices.Count() >= n_days) { + double std_dev = MathNet.Numerics.Statistics.Statistics.StandardDeviation(last_n_close_prices); + rv.setValue("std_dev", std_dev); + } + //rv.setValue("id", id++); + return rv; + }; + ViewStream rs = new GlobalMapStream(ohlcv_str, "ts_event", find_std_dev); + rs = new NullFilterStream(rs, "std_dev", true); + mySetRealSource(rs); + } + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs index 57392b55b..c4a7d1ec0 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/VWAPStream.cs @@ -3,6 +3,7 @@ namespace bts.SilverPine { using BCG; using MongoDB.Driver.Linq; using System.ComponentModel.DataAnnotations; + using ZstdSharp.Unsafe; [DataStreamUserRespresentation("VWAP")] public class VWAPStream : ViewStream { @@ -17,7 +18,7 @@ namespace bts.SilverPine { IBlueRow ir = rows.ElementAt((int)idx); var sel_rows = rows.Skip((int)idx).Where(r => r.get_datetime("ts_event") >= end_dt && r.get_datetime("ts_event") <= begin_dt); var volumeprices = sel_rows.Select(r => Tuple.Create(r.get("volume"), r.get("close"))).ToList(); - double vwap = ChartingHelper.calculate_VWAP(volumeprices); + double vwap = IndicatorCalc.calculate_VWAP(volumeprices); IBlueTableSchema schema = ir.schema.add_columns(Tuple.Create(typeof(double), "vwap")); MutableBlueRow nr = new MutableBlueRow(schema, ir); //vwap = -313.987; @@ -26,15 +27,25 @@ namespace bts.SilverPine { }); return rs; } + // the args want dt1 > dt2, if its nto we, swap them + static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) { + if (dt1 < dt2) { + DateTime aux = dt1; + dt1 = dt2; + dt2 = aux; + } + } + // this assumes the ohlc_str has diff contracts public VWAPStream(string symbol_or_contract, DateTime begin_dt, DateTime end_dt) { + swap_if_necessary(ref begin_dt, ref end_dt); Tuple ccat_sym_contr = ContractUtils.get_symbol_category(symbol_or_contract); string symbol = String.Empty; symbol = ccat_sym_contr.Item2; var ohlcv_params = OHLCVStream.get_params(begin_dt, end_dt); ViewStream vwap_str = build_vwap_stream(symbol, begin_dt, end_dt); // vwap_str now has vwap for all contracts. if the arg was a symbol, we need to filter it for the active contract, - // whihc we do with FutureActivesContractStream. If the arg was a ocntract we'll simply filter it + // whihc we do with FutureActivesContractStream. If the arg was a contract we'll simply filter it ViewStream rs = vwap_str; switch (ccat_sym_contr.Item1) { case ContractUtils.ContractCategory.FutureString: @@ -46,6 +57,7 @@ namespace bts.SilverPine { break; } rs = new FilterStream(rs, r => r.get_datetime("ts_event").CompareTo(begin_dt) == 0); + rs = new NewProjectStream(rs, "! vwap"); mySetRealSource(rs); } diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs new file mode 100644 index 000000000..6816cd940 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs @@ -0,0 +1,18 @@ +namespace bts.SilverPine { + using BCG; + + [DataStreamUserRespresentation("Events")] + public class EventsStream : ViewStream { + public EventsStream(string event_type) : this(ViewStreamDatabase.instance, event_type) { } + + public EventsStream(ViewStreamDatabase vsd, string event_type) { + ViewStream rs = + new SQLiteStream("Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;", + "select * from events"); + if (!String.IsNullOrEmpty(event_type)) { + rs = new FilterStream(rs, r => String.Compare(event_type, r.get("event_type"), true) == 0); + } + mySetRealSource(rs); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesRollStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesRollStream.cs index 01783ee8a..9115f3731 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesRollStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesRollStream.cs @@ -10,6 +10,7 @@ namespace bts.SilverPine { //ViewStream ohlcv_str = new OHLCVStream(future_symbol); ViewStream rs = new FuturesActiveContractStream(future_symbol); rs = new ColumnEvolutionStream(rs, "ts_event", "contract"); + rs = new NewProjectStream(rs, "ts_event:roll_date !"); mySetRealSource(rs); } } diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVPriceDiffStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVPriceDiffStream.cs index f5fc9dfd4..61f9983a5 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVPriceDiffStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVPriceDiffStream.cs @@ -29,6 +29,7 @@ namespace bts.SilverPine { ViewStream rs = _get_ohlcv_steam(symbol_or_contract, d1, d2); rs = new ColumnDifferenceStream(rs, r => d1.Equals(r.get_datetime("ts_event")), r => d2.Equals(r.get_datetime("ts_event")), col, col); + rs = new NewProjectStream(rs, "diff !"); mySetRealSource(rs); } diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs index 35c00fe6c..383a7becb 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/SettlementPriceStream.cs @@ -45,7 +45,7 @@ namespace bts.SilverPine { return dt <= begin_dt && dt >= end_dt; }); rs = new MathStream(rs, "settlement = close * 1.0"); - rs = new NewProjectStream(rs, "ts_event settlement"); + rs = new NewProjectStream(rs, " ! ts_event:date settlement"); mySetRealSource(rs); } diff --git a/src/DataStreamSuite/SharedStreams/NL/FewShotPrompts.json b/src/DataStreamSuite/SharedStreams/NL/FewShotPrompts.json index 1c7cc8558..96209e815 100644 --- a/src/DataStreamSuite/SharedStreams/NL/FewShotPrompts.json +++ b/src/DataStreamSuite/SharedStreams/NL/FewShotPrompts.json @@ -3,7 +3,7 @@ { "category": "Settlement", "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is Settlement (symbol_or_contract, datetime1). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second argument is the date at which the settlement is to be fetched. There is an overloaded function, Settlement (symbol_or_contract, begin_dt, end_dt) which gets the settlements for all dates from begin_dt to end_dt. This will also have datetimes expressed as Today, Yesterday, Last Friday, First Friday of the month, Last Thursday of August etc. You will need to know what Today is, since there will be referecnes to last week, yesterday etc. These examples are assuming Today is 10/11/2024, however you should use the Today's real date", - "prompt": "What was the settlement for CL on June 8th?", + "prompt": "What was the settlement for SP 500 Future on June 8th?", "examples": [ { "natural_language": "Find Settlement price of the SP500 future on 04/22/2023.", @@ -42,76 +42,76 @@ { "category": "VWAP", "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is VWAP (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil, or ES for SP 500) or a specific future contract like crude oil Dec 24 future (CLZ4) or SP 500 Dec 2024 (ESZ4). The second argument is where the VWAP has to be calculated (also the beginning of the time window), and the third argument is datetime specifying the end of the time window. So, for example, a 20-day VWAP on 9/30/2023, the first argument will be 9/30/2023, and the third argument will be 9/10/2023 (20 days prior). This will also have datetimes expressed as Today, Yesterday, Last Friday, Last Sunday, etc. For the purpose of this few-shot prompts, let's assume today is 10/11/2024.", - "prompt": "What was the VWAP for ZN over the past 30 days?", + "prompt": "What was the VWAP for SP 500 Future over the past 30 days?", "examples": [ { "natural_language": "Find the 20 day VWAP of the SP future on 04/22/2023.", - "dsl": "VWAP(\"ES\", \"4/22/2023\", \"4/02/2023\")" + "dsl": "VWAP(\"ES\", 4/22/2023, 4/02/2023)" }, { "natural_language": "Find Yesterday's 10 day VWAP of the Crude Oil Future.", - "dsl": "VWAP(\"CL\", \"10/10/2024\", \"10/1/2024\")" + "dsl": "VWAP(\"CL\", 10/10/2024, 10/1/2024)" }, { "natural_language": "Find today's 30-day VWAP of the 10 year treasury future.", - "dsl": "VWAP(\"ZN\", \"10/11/2024\", \"9/11/2024\")" + "dsl": "VWAP(\"ZN\", 10/11/2024, 9/11/2024)" }, { "natural_language": "Find the 10 hour VWAP of the Natural Gas future from yesterday at 5:00 PM.", - "dsl": "VWAP(\"NG\", \"9/27/2024 17:00:00\", \"9/27/2024 7:00:00\")" + "dsl": "VWAP(\"NG\", 9/27/2024 17:00:00, 9/27/2024 7:00:00)" } ] }, { "category": "Price Diff", "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is OHLCVPriceDiff (symbol_or_contract, datetime1, datetime2). The first argument is a future (like CL, for crude oil) or a specific future contract like crude oil Dec 24 future (CLZ4). The second and third arguments are dates, that the function finds the difference of prices at. Assume today is 10/15/2024", - "prompt": "What is the difference in ZN price from July 10th to July 14th?", + "prompt": "What is the difference in SP 500 Future from July 10th to July 14th?", "examples": [ { "natural_language": "What is the difference in ZN price from July 10th to July 14th?", - "dsl": "OHLCVPriceDiff(\"ZN\", \"7/14/2024\", \"7/10/2024\")" + "dsl": "OHLCVPriceDiff(\"ZN\", 7/14/2024, 7/10/2024)" }, { "natural_language": "What is the price difference in SP Future price from Sept 10th and Sept 5th?", - "dsl": "OHLCVPriceDiff(\"ES\", \"9/10/2024\", \"9/5/2024\")" + "dsl": "OHLCVPriceDiff(\"ES\", 9/10/2024, 9/5/2024)" }, { "natural_language": "What is the price difference in Natural Gas Dec Future price from Aug 30th and Aug 20th?", - "dsl": "OHLCVPriceDiff(\"NGZ4\", \"8/30/2024\", \"8/20/2024\")" + "dsl": "OHLCVPriceDiff(\"NGZ4\", 8/30/2024, 8/20/2024)" } ] }, { - "category": "Abs Volume", - "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is AbsVolume(symbol, ContractType, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc). If no symbol is specified, use \"*\". The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment. For this prompt, assume today is 10/11/2024 ", - "prompt": "Which SP Future traded the most between 7/14/2024 and 7/10/2024", + "category": "Absolute Volume", + "description": "this is a few shot learning example for an NL to DSL. The DSL function in question is AbsVolume(symbol, ContractType, datetime1, datetime2), which computes the highest volume in the date range, specified by datetime1, and datetime2. The first argument is a future symbol (like CL for crude oil, ES for SP 500 etc). The second argument is the ContractType, which is \"Future\" for futures and \"FutureSpread\" for future spreads. the 3rd argument is the begin date and the 4th argument is the end date, between which the max volume is computed. If no future is specified use \"*\" from the first arugment. For this prompt, assume today is 10/11/2024 ", + "prompt": "Which SP 500 Future traded the most between 7/14/2024 and 7/10/2024", "examples": [ { "natural_language": "Which SP Future traded the most between 7/14/2024 and 7/10/2024", - "dsl": "AbsVolume(\"ES\", \"Future\", \"7/14/2024\", \"7/10/2024\")" + "dsl": "AbsVolume(\"ES\", \"Future\", 7/14/2024, 7/10/2024)" }, { "natural_language": "Which Gold Spread traded the most between Today and Sept 20th", - "dsl": "AbsVolume(\"GC\", \"FutureSpread\", \"10/1/2024\", \"9/20/2024\")" + "dsl": "AbsVolume(\"GC\", \"FutureSpread\", 10/1/2024, 9/20/2024)" }, { "natural_language": "Which 10 year Spread traded the highest volume last week?", - "dsl": "AbsVolume(\"ZN\", \"FutureSpread\", \"9/27/2024\", \"9/23/2024\")" + "dsl": "AbsVolume(\"ZN\", \"FutureSpread\", 9/27/2024, 9/23/2024)" }, { "natural_language": "Which Corn Spread traded the most today?", - "dsl": "AbsVolume(\"ZC\", \"FutureSpread\", \"10/1/2024\", \"10/1/2024\")" + "dsl": "AbsVolume(\"ZC\", \"FutureSpread\", 10/1/2024, 10/1/2024)" }, { - "natural_language": "Which Future traded the most today?", - "dsl": "AbsVolume(\"*\", \"Future\", \"9/30/2024\", \"9/30/2024\")" + "natural_language": "Which Gold Future traded the most today?", + "dsl": "AbsVolume(\"GC\", \"Future\", 9/30/2024, 9/30/2024)" } ] }, { "category": "Roll", "description": "This is a few shot learning example for an NL to DSL. The DSL function in question is FuturesRoll(symbol) which computes the rollover date for a future, ie when the highest volume swtches from one expiration to the next one. ", - "prompt": "When does the roll typically occur In CL?", + "prompt": "When does the roll typically occur In SP 500?", "examples": [ { "natural_language": "When does the roll typically occur In CL?", @@ -125,13 +125,9 @@ "natural_language": "When does the roll usually occur for Gold Future", "dsl": "FuturesRoll(\"GC\")" }, - { - "natural_language": "Which Corn Spread traded the most today?", - "dsl": "AbsVolume(\"ZC\", \"FutureSpread\", \"10/1/2024\", \"10/1/2024\")" - }, { "natural_language": "At what date does the future rollover happens for SP500?", - "dsl": "FuturesRoll(\"SI\")" + "dsl": "FuturesRoll(\"ES\")" }, { "natural_language": "When does the future rollover happens for the Corn Future?", @@ -141,8 +137,8 @@ }, { "category": "Daily Move", - "description": "This is a few shot learning example for an NL to DSL. For the purpose of this few shot prompts, lets assume today is 10/3/2024 ", - "prompt": "How often did crude settle + or - 10% over the previous day?", + "description": "This is a few shot learning example for an NL to DSL. ", + "prompt": "How often did Sp 500 Future settle + or - 10% over the previous day?", "examples": [ { "natural_language": "List all the days when the SP 500 closed + or -5% from the previous day's close", @@ -150,22 +146,22 @@ }, { "natural_language": "List all the days when the Crude Oil closed + or -2.5% from the previous day's close", - "dsl": "Average(filteredOHLCV(\"CL\", \"10/02/2023\", \"10/01/2023\"), \"volume\")" + "dsl": "ColDiff(filteredOHLCV(\"CL\"),\"ts_event\", -1, \"close\", \"close\", \"close_diff\", 2.5)" }, { "natural_language": "Show me all the days when the Ten year Treaury Future opened + or -1% from the previous day's open", - "dsl": "Average(filteredOHLCV(\"SI\", \"10/03/2023\", \"9/13/2023\"), \"volume\")" + "dsl": "ColDiff(filteredOHLCV(\"ZN\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)" }, { "natural_language": "List all the trading days when the Silver Future opened + or -1% from the previous day's close", - "dsl": "Average(filteredOHLCV(\"ES\", \"10/03/2023\", \"9/13/2023\"), \"volume\")" + "dsl": "ColDiff(filteredOHLCV(\"SI\"),\"ts_event\", -1, \"open\", \"open\", \"open_diff\", 1.0)" } ] }, { "category": "Average Volume", "description": "This is a few shot learning example for an NL to DSL. For the purpose of this few shot prompts, lets assume today is 10/3/2024 ", - "prompt": "When does the most open interest typically shift from one contract to another in e-minis?", + "prompt": "When does the most open interest typically shift from one contract to another in SP 500 Future?", "examples": [ { "natural_language": "When does the most open interest typically shift from one contract to another in e-minis? ", @@ -176,6 +172,52 @@ "dsl": "FuturesRoll(\"CL\")" } ] + }, + { + "category": "Events", + "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show the price of a future (treasury future, or SP 500 future etc) around an event type. The event type could be \"FOMC Announcement\" or \"Storage numbers\" etc. The DSL is as follows: EventsPriceAction(future_name, event_type, pre_event_window, post_event_window, number_of_events) first arg is the name of the future, (ex ES or ZN etc), the second argument is the event type, for example, \"FOMC announcement\", the 3rd argument is a time window for how much pre-event data to include in the end result, the 4th argument is how much post-event data to include in the end result. Both are strings of the form \"5d\" for 5 days, \"5h\" for 5 hours and \"120m\" for 120 minutes and so on. Usually, only one of the event periods is provided. Set the other to be half of what the user provided. The last argument is the number of events to consider. So if it’s 7, that means the most recent 7 events. If the user does not specify that, pass a large number.", + "prompt": "What did the volume and price of SP500 Future do following the release of the last 10 FOMC rates?", + "examples": [ + { + "natural_language": "How did SP 500 Future do in the 5 days following the FOMC announcements?", + "dsl": "EventsPriceAction(\"ES\", \"FOMC Announcement\", \"2d\", \"5d\", 10)" + }, + { + "natural_language": "How did the price of Treasury Future evolve in the 2 days following the Job Reports?", + "dsl": "EventsPriceAction(\"ZN\", \"Job Reports\", \"1d\", \"2d\", 10)" + }, + { + "natural_language": "Show me the price movement and volume of Crude Oil futures during the 5-minute window following the last 5 OPEC meetings.", + "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"2m\", \"5m\", 5)" + }, + { + "natural_language": "Show me the price movement and volume of SP 500 futures preceding the 10-minute window of the last 5 OPEC meetings.", + "dsl": "EventsPriceAction(\"CL\", \"OPEC Meeting\", \"10m\", \"5m\", 5)" + } + ] + }, + { + "category": "Indicator, Standard Deviation", + "description": "This is a few shot learning example for an NL to DSL. In the NL prompt the user is asking to show all the days when the price of a future (treasury future, or SP 500 future etc) moved more than m times the N days standard deviation, where both m and N are provided. For example, the DSL for \"Using a 90d standard deviation, give me the dates and amounts of the last 4+ standard deviation moves in WTI futures\" would be: r=standardDeviation(\"ES\", \"90d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 4\")\nr=filter(r,\"abs_daily_change >= comp\"). The first line, creates a stream for StandardDeviation, based on the number of days, for example 90d standard deviation, and assigned it to r, the second line takes r from the first line and passes it to a math stream that creates 2 new columns, abs_daily_change and comp. comp is the factor (4, from 4+ in the prompt) times standard deviation. The third line filters (like a sql where) of rows where abs_daily_change >= comp", + "prompt": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in SP futures ?", + "examples": [ + { + "natural_language": "Using a 90d standard deviation, give me the dates and amounts of the last 1+ standard deviation moves in WTI futures ?", + "dsl": "r=standardDeviation(\"CL\", \"90d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.0\")\nr=filter(r,\"abs_daily_change >= comp\")" + }, + { + "natural_language": "Show me the times the SP Futures moved twice the 60 day standard deviation", + "dsl": "r=standardDeviation(\"ES\", \"60d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")" + }, + { + "natural_language": "List all the days the Gold Futures moved 1.5 times the 45 day standard deviation", + "dsl": "r=standardDeviation(\"GC\", \"45d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 1.5\")\nr=filter(r,\"abs_daily_change >= comp\")" + }, + { + "natural_language": "List all the days the 10 year treasury future moved more than twice the 1 month standard deviation", + "dsl": "r=standardDeviation(\"ZN\", \"30d\")\nr=math(r, \"abs_daily_change = Abs(close - open) comp = std_dev * 2.0\")\nr=filter(r,\"abs_daily_change >= comp\")" + } + ] } ] } \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/NL/NLToFewShotPromptStream.cs b/src/DataStreamSuite/SharedStreams/NL/NLToFewShotPromptStream.cs index 5c1ef23bb..4e0f282fd 100644 --- a/src/DataStreamSuite/SharedStreams/NL/NLToFewShotPromptStream.cs +++ b/src/DataStreamSuite/SharedStreams/NL/NLToFewShotPromptStream.cs @@ -10,167 +10,13 @@ namespace bts.SilverPine { // given an nl prompt, retreives the correct fewshot prompt and converts it into a form // that be sent to openAI as a prompt public class NLToFewShotPromptStream : ViewStream { - - // Represents a pair of a natural language query and its dsl - public class FewShotExample { - public string natural_language { get; set; } - public string dsl { get; set; } - } - - // Class to represent each prompt category - public class FewShotPrompt { - public string category { get; set; } - // this is the explaination of the DSL - public string description { get; set; } - // th actual prompt given by the user/CME - public string prompt { get; set; } - public List examples { get; set; } - - public override string ToString() { - string rv = $"System: {description}\n"; - int ex_no = 0; - foreach (FewShotExample ex in examples) { - rv += $"{++ex_no}. Natural Language: \"{ex.natural_language}\"" + "\n"; - rv += $" DSL: {ex.dsl}\n"; - - } - return rv; - } - } - - // this is how it should be prepended to the actual prompt: - // System: This is a few-shot learning example for converting natural language prompts to a DSL function for settlement data retrieval. The DSL function can take either a single date or a date range. Assume today's date is 10/11/2024. The function is structured as follows: - // - Settlement(symbol_or_contract, datetime1): Retrieves the settlement for the specified date. - // - Settlement(symbol_or_contract, begin_dt, end_dt): Retrieves the settlements for all dates in the specified range. - - // Examples: - // 1. Natural Language: "Find the Settlement price of the SP future on 04/22/2023." - // DSL: Settlement(SP, 4/22/2023) - - // 2. Natural Language: "Find Yesterday's Settlement of the Crude Oil Future." - // DSL: Settlement(CL, 10/10/2024) - - // 3. Natural Language: "Find today's settlement of the 10-year treasury future." - // DSL: Settlement(ZN, 10/10/2024) - - // 4. Natural Language: "Find the Settlement of Natural Gas future on the first Friday of Sept." - // DSL: Settlement(NG, 9/6/2024) - - // 5. Natural Language: "Find the Settlement of Natural Gas Dec 24 future in the first Week of October." - // DSL: Settlement(NGZ4, 10/1/2024, 10/1/2024) - - // 6. Natural Language: "Find the settlement prices for the Wheat and Corn spread for the first two weeks of November." - // DSL: Settlement(ZWZ4-ZCZ4, 11/1/2024, 11/14/2024) - - // [Your Real Prompt Goes Here] - - - // Root class to map the JSON structure - public class Root { - public List prompts { get; set; } - } - - public static Root read_json_file() { - // TODO - string json_file_path = @"C:\Users\m_tah\development\spark\src\DataStreamSuite\SharedStreams\NL\FewShotPrompts.json"; - string json_string = File.ReadAllText(json_file_path); - var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; - Root root = JsonSerializer.Deserialize(json_string, options); - return root; - } - - private readonly string _nl_prompt; - // this is the first stream that sends all FewShotPrompt.prompt and asks the LLM - // to find the FewShotPrompt.prompt that matches the nl_prompt the closest. That - // gives us our FewShotPrompt - private ViewStream _nl_to_fewshot_index_stream; public NLToFewShotPromptStream(string nl) { - _nl_prompt = nl; - } - - private static string _build_fewshot_prompt(FewShotPrompt fw_prompt, string nl_prompt) { - string rv = "You are a helpful assistant that converts natural language into DSL. "; - rv += $"{fw_prompt}\n\n"; - rv += "Now, convert the following prompt into DSL: \n"; - rv += $"Natural Language: {nl_prompt}\n"; - return rv; - } - - protected override void myOnFirstSubscriber() { - Root root = read_json_file(); - //FewShotPrompt p = find_matching_prompt(root, _nl_prompt); - //string final_prompt = _build_final_prompt(p, _nl_prompt); - string meta_prompt = build_meta_prompt(root, _nl_prompt); - _nl_to_fewshot_index_stream = new OpenAIPromptStream(meta_prompt); - _nl_to_fewshot_index_stream.onChange += _on_fewshot_index; - //Log.Gen.info($"NLToDSLStream.myOnFirstSubscriber meta_prompt: {meta_prompt}"); - - // this needs to be sent to the LLM to get the few-shot item. Once the few-shot item is received, then - // we can prepend the fewshot to the prompt and send it - //Log.Gen.info($"NLToDSLStream.myOnFirstSubscriber total_prompt: {final_prompt}"); - //_fire_event(final_prompt); - } - - private void _on_fewshot_index(object dp_obj) { - DataPacket dp = dp_obj as DataPacket; - IBlueRow r = dp.newRows.to_ienum().ElementAt(0); - string response = r.get("response"); - Log.Gen.info($"NLToDSLStream._on_fewshot_index prompt: {r.get("prompt")}"); - Log.Gen.info($"NLToDSLStream._on_fewshot_index response: {response}"); - string[] toks = response.Split('.'); - int index = Int32.Parse(toks[0]); - var root = read_json_file(); - FewShotPrompt fs_prompt = root.prompts[index]; - string final_prompt = _build_fewshot_prompt(fs_prompt, _nl_prompt); - _fire_event(final_prompt); - } - - private void _fire_event(string final_prompt) { - IBlueTableSchema schema = BlueDataUtility.form_schema("NLToFewShotPromptStream", - Tuple.Create("id", TypeCode.Int32), - Tuple.Create("prompt", TypeCode.String), - Tuple.Create("few_shot_prompt", TypeCode.String)); - IBlueRow nr = new BlueRow(schema, 0, _nl_prompt, final_prompt); - BlueTable nt = new BlueTable(schema); - nt.addRow(nr); - var dp_out = BlueDataUtility.build_dp(nt, myLastValue as DataPacket); - myFireEvent(dp_out); - } - - // since find_matching_prompt using string matching loses semantics and often results - // the incorrect few-shot entry (ex for, "Find the VWAP of the Sp 500 Dec 24 future during the first Week of October 24?"). - // so we assemble all the prompts from the few-shot (field "prompt") entry and ask an LLM to tell us which is the closest to the prompt. - // based on that we select the few-shot entry. This is what this function does - public static string build_meta_prompt(Root root, string nl_prompt) { - string meta_promp = "Take a look at the following prompt examples:\n"; - int rv = 0; - foreach (FewShotPrompt fs_entry in root.prompts) { - meta_promp += $"{rv++}. {fs_entry.prompt}\n"; - } - meta_promp += "Based on the above prompts, tell me which prompt (give me answer as the index only, and nothing else) does the following prompt most closely matches to: \n"; - meta_promp += nl_prompt; - return meta_promp; - } - - public static FewShotPrompt find_matching_prompt(Root root, string nl_prompt) { - var levenshtein = new SimMetrics.Net.Metric.Levenstein(); - FewShotPrompt rv = root.prompts.ElementAt(0); - Func get_total_sim = (fsp) => { - double similiarity = levenshtein.GetSimilarity(fsp.prompt, nl_prompt); - foreach (FewShotExample fs_ex in fsp.examples) { - similiarity += levenshtein.GetSimilarity(fs_ex.natural_language, nl_prompt); - } - return similiarity; - }; - double highest_simmiliarity = 0.0; - foreach (FewShotPrompt p in root.prompts) { - double similiarity = get_total_sim(p); //levenshtein.GetSimilarity(p.prompt, nl_prompt); - if (similiarity > highest_simmiliarity) { - highest_simmiliarity = similiarity; - rv = p; - } - } - return rv; + // TODO based on confi, we can either have NLToIndexedFewShotPromptStream + // or NLToNonIndexedFewShotPromptStream (which is when a al the few shots are + // concatenated and sent) + mySetRealSource(new NLToIndexedFewShotPromptStream(nl)); + //_nl_prompt = nl; } } + } \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/NL/NLToIndexedFewShotPromptStream.cs b/src/DataStreamSuite/SharedStreams/NL/NLToIndexedFewShotPromptStream.cs new file mode 100644 index 000000000..e8f3fbd8c --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/NL/NLToIndexedFewShotPromptStream.cs @@ -0,0 +1,197 @@ +namespace bts.SilverPine { + using BCG; + + using MongoDB.Driver.Linq; + + using SimMetrics.Net.Metric; + + using System.Text.Json; + + + [DataStreamUserRespresentation("nl_to_indexed_fewshot_prompt")] + // given an nl prompt, retreives the correct fewshot prompt and converts it into a form + // that be sent to openAI as a prompt + public class NLToIndexedFewShotPromptStream : ViewStream { + + // Represents a pair of a natural language query and its dsl + public class FewShotExample { + public string natural_language { get; set; } + public string dsl { get; set; } + } + + // Class to represent each prompt category + public class FewShotPrompt { + public string category { get; set; } + // this is the explaination of the DSL + public string description { get; set; } + // th actual prompt given by the user/CME + public string prompt { get; set; } + public List examples { get; set; } + + public override string ToString() { + string rv = $"System: {description}\n"; + int ex_no = 0; + foreach (FewShotExample ex in examples) { + rv += $"{++ex_no}. Natural Language: \"{ex.natural_language}\"" + "\n"; + rv += $" DSL: {ex.dsl}\n"; + + } + return rv; + } + } + + // this is how it should be prepended to the actual prompt: + // System: This is a few-shot learning example for converting natural language prompts to a DSL function for settlement data retrieval. The DSL function can take either a single date or a date range. Assume today's date is 10/11/2024. The function is structured as follows: + // - Settlement(symbol_or_contract, datetime1): Retrieves the settlement for the specified date. + // - Settlement(symbol_or_contract, begin_dt, end_dt): Retrieves the settlements for all dates in the specified range. + + // Examples: + // 1. Natural Language: "Find the Settlement price of the SP future on 04/22/2023." + // DSL: Settlement(SP, 4/22/2023) + + // 2. Natural Language: "Find Yesterday's Settlement of the Crude Oil Future." + // DSL: Settlement(CL, 10/10/2024) + + // 3. Natural Language: "Find today's settlement of the 10-year treasury future." + // DSL: Settlement(ZN, 10/10/2024) + + // 4. Natural Language: "Find the Settlement of Natural Gas future on the first Friday of Sept." + // DSL: Settlement(NG, 9/6/2024) + + // 5. Natural Language: "Find the Settlement of Natural Gas Dec 24 future in the first Week of October." + // DSL: Settlement(NGZ4, 10/1/2024, 10/1/2024) + + // 6. Natural Language: "Find the settlement prices for the Wheat and Corn spread for the first two weeks of November." + // DSL: Settlement(ZWZ4-ZCZ4, 11/1/2024, 11/14/2024) + + // [Your Real Prompt Goes Here] + + + // Root class to map the JSON structure + public class Root { + public List prompts { get; set; } + } + + public static Root read_json_file() { + // TODO + string json_file_path = @"C:\Users\m_tah\development\spark\src\DataStreamSuite\SharedStreams\NL\FewShotPrompts.json"; + string json_string = File.ReadAllText(json_file_path); + var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; + Root root = JsonSerializer.Deserialize(json_string, options); + return root; + } + + private readonly string _nl_prompt; + private string _fewshot_index_prompt; + private string _fewshot_index_reponse; + // this is the first stream that sends all FewShotPrompt.prompt and asks the LLM + // to find the FewShotPrompt.prompt that matches the nl_prompt the closest. That + // gives us our FewShotPrompt + private ViewStream _nl_to_fewshot_index_stream; + public NLToIndexedFewShotPromptStream(string nl) { + _nl_prompt = nl; + } + + private static string _build_fewshot_prompt(FewShotPrompt fw_prompt, string nl_prompt) { + string rv = "You are a helpful assistant that converts natural language into DSL. "; + rv += $"{fw_prompt}\n\n"; + rv += "Now, convert the following prompt into DSL: \n"; + rv += $"Natural Language: {nl_prompt}\n"; + return rv; + } + + protected override void myOnFirstSubscriber() { + Root root = read_json_file(); + string meta_prompt = build_fewshot_index_prompt(root, _nl_prompt); + _fewshot_index_prompt = meta_prompt; + Log.Gen.info($"NLToDSLStream.myOnFirstSubscriber _fewshot_index_prompt: {_fewshot_index_prompt}"); + _nl_to_fewshot_index_stream = new OpenAIPromptStream(meta_prompt); + _nl_to_fewshot_index_stream.onChange += _on_fewshot_index; + //Log.Gen.info($"NLToDSLStream.myOnFirstSubscriber meta_prompt: {meta_prompt}"); + + // this needs to be sent to the LLM to get the few-shot item. Once the few-shot item is received, then + // we can prepend the fewshot to the prompt and send it + //Log.Gen.info($"NLToDSLStream.myOnFirstSubscriber total_prompt: {final_prompt}"); + //_fire_event(final_prompt); + } + + private void _on_fewshot_index(object dp_obj) { + DataPacket dp = dp_obj as DataPacket; + IBlueRow r = dp.newRows.to_ienum().ElementAt(0); + string response = r.get("response"); + _fewshot_index_reponse = response; + Log.Gen.info($"NLToDSLStream._on_fewshot_index prompt: {r.get("prompt")}"); + Log.Gen.info($"NLToDSLStream._on_fewshot_index response: {response}"); + // parse the response + // the response is mostly like : + // 0. 0. What was the settlement for SP 500 Future on June 8th? + // however, sometimes for no reason is returns a + // Prompt 0: What was the settlement for SP 500 Future on June 8th? + int index = 0; + if (response.ToLower().StartsWith("prompt")) { + // for the unusual case + string[] toks = response.Split(':', '.'); + toks = toks[0].Split(' '); + index = Int32.Parse(toks[1]); + } else { + // for the usual case + string[] toks = response.Split('.'); + index = Int32.Parse(toks[0]); + } + var root = read_json_file(); + FewShotPrompt fs_prompt = root.prompts[index]; + string final_prompt = _build_fewshot_prompt(fs_prompt, _nl_prompt); + _fire_event(final_prompt); + } + + private void _fire_event(string final_prompt) { + IBlueTableSchema schema = BlueDataUtility.form_schema("NLToIndexedFewShotPromptStream", + Tuple.Create("id", TypeCode.Int32), + Tuple.Create("main_prompt", TypeCode.String), + Tuple.Create("fewshot_index_prompt", TypeCode.String), + Tuple.Create("fewshot_index_response", TypeCode.String), + Tuple.Create("few_shot_prompt", TypeCode.String)); + IBlueRow nr = new BlueRow(schema, 0, _nl_prompt, _fewshot_index_prompt, _fewshot_index_reponse, final_prompt); + BlueTable nt = new BlueTable(schema); + nt.addRow(nr); + var dp_out = BlueDataUtility.build_dp(nt, myLastValue as DataPacket); + myFireEvent(dp_out); + } + + // since find_matching_prompt using string matching loses semantics and often results + // the incorrect few-shot entry (ex for, "Find the VWAP of the Sp 500 Dec 24 future during the first Week of October 24?"). + // so we assemble all the prompts from the few-shot (field "prompt") entry and ask an LLM to tell us which is the closest to the prompt. + // based on that we select the few-shot entry. This is what this function does + public static string build_fewshot_index_prompt(Root root, string nl_prompt) { + string meta_promp = "Take a look at the following prompt examples:\n"; + int rv = 0; + foreach (FewShotPrompt fs_entry in root.prompts) { + meta_promp += $"{rv++}. {fs_entry.prompt}\n"; + } + meta_promp += "Based on the above prompts, tell me which prompt (give me answer as the index only, and nothing else) does the following prompt most closely matches to: \n"; + meta_promp += nl_prompt; + return meta_promp; + } + + public static FewShotPrompt find_matching_prompt(Root root, string nl_prompt) { + var levenshtein = new SimMetrics.Net.Metric.Levenstein(); + FewShotPrompt rv = root.prompts.ElementAt(0); + Func get_total_sim = (fsp) => { + double similiarity = levenshtein.GetSimilarity(fsp.prompt, nl_prompt); + foreach (FewShotExample fs_ex in fsp.examples) { + similiarity += levenshtein.GetSimilarity(fs_ex.natural_language, nl_prompt); + } + return similiarity; + }; + double highest_simmiliarity = 0.0; + foreach (FewShotPrompt p in root.prompts) { + double similiarity = get_total_sim(p); //levenshtein.GetSimilarity(p.prompt, nl_prompt); + if (similiarity > highest_simmiliarity) { + highest_simmiliarity = similiarity; + rv = p; + } + } + return rv; + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/NL/OpenAIPromptStream.cs b/src/DataStreamSuite/SharedStreams/NL/OpenAIPromptStream.cs index 84b148d7f..b1f577c55 100644 --- a/src/DataStreamSuite/SharedStreams/NL/OpenAIPromptStream.cs +++ b/src/DataStreamSuite/SharedStreams/NL/OpenAIPromptStream.cs @@ -82,6 +82,7 @@ namespace bts.SilverPine { } public OpenAIPromptStream(ViewStreamDatabase vsd, string prompt) { + Log.Gen.info($"OpenAIPromptStream.OpenAIPromptStream prompt: {prompt}"); _prompt = prompt; _vsd = vsd; }