diff --git a/src/DataStreamSuite/DataStream/BlueOrb/AddColumnsStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/AddColumnsStream.cs new file mode 100644 index 000000000..3495120b8 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/AddColumnsStream.cs @@ -0,0 +1,11 @@ +namespace bts.BlueOrb { + using BCG; + + [DataStreamUserRespresentation("AddColumnsStream")] + public class AddColumnsStream: ViewStream { + public AddColumnsStream(ViewStream parent_str, string add_col_expr) { + ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr); + mySetRealSource(rs); + } + } +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs index 115b025aa..0555ed628 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/CountStream.cs @@ -1,7 +1,7 @@ using System; namespace BCG { - [DataStreamUserRespresentation("count")] + [DataStreamUserRespresentation("CountStream")] public class CountStream : ViewStream { private DataPacket myLastFiredDataPacket; private IBlueTableSchema mySchema = new BlueTableSchema("count", diff --git a/src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs index b91b30944..5e7f5702e 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs @@ -3,36 +3,34 @@ namespace bts.BlueOrb { using System; using System.Text.RegularExpressions; - - public class RegExpFilter : IStreamFilter { - private Regex[] myRegExps; - private string[] myCols; - - public RegExpFilter(string col, string regExp) - : this(new string[] { col }, new string[] { regExp }) { } - - public RegExpFilter(string[] cols, string[] regExps) { - myCols = cols; - myRegExps = new Regex[regExps.Length]; - for (int i = 0; i < regExps.Length; i++) - myRegExps[i] = new Regex(regExps[i]); - } - - #region IStreamFilter Members - public bool filter(IBlueRow dr) { - bool rv = true; - for (int i = 0; i < myCols.Length; i++) { - rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString()); - if (!rv) - return rv; - } - return rv; - } - #endregion - } - - //[DataStreamUserReprsentation("FilterStream")] + [DataStreamUserRespresentation("FilterStream")] public class FilterStream : ViewStream { + public class RegExpFilter : IStreamFilter { + private Regex[] myRegExps; + private string[] myCols; + + public RegExpFilter(string col, string regExp) + : this(new string[] { col }, new string[] { regExp }) { } + + public RegExpFilter(string[] cols, string[] regExps) { + myCols = cols; + myRegExps = new Regex[regExps.Length]; + for (int i = 0; i < regExps.Length; i++) + myRegExps[i] = new Regex(regExps[i]); + } + + #region IStreamFilter Members + public bool filter(IBlueRow dr) { + bool rv = true; + for (int i = 0; i < myCols.Length; i++) { + rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString()); + if (!rv) + return rv; + } + return rv; + } + #endregion + } private BlueTable myLastFiredRows; private IBlueRowCollection myLastReceivedRows; private IStreamFilter myFilter; diff --git a/src/DataStreamSuite/DataStream/BlueOrb/Join/INonIncJoinInlineMapper.cs b/src/DataStreamSuite/DataStream/BlueOrb/Join/INonIncJoinInlineMapper.cs deleted file mode 100644 index 90d3c5e62..000000000 --- a/src/DataStreamSuite/DataStream/BlueOrb/Join/INonIncJoinInlineMapper.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BCG; - -namespace bts.BlueOrb { - public interface INonIncJoinInlineMapper - { - IBlueTableSchema get_output_schema(IBlueTableSchema schema); - - IBlueRowCollection map_rows(IBlueRowCollection rows); - - event MapperChangeDelegate onMapperChange; - } -} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinBookkeeper.cs b/src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinBookkeeper.cs deleted file mode 100644 index 947a8a4ce..000000000 --- a/src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinBookkeeper.cs +++ /dev/null @@ -1,152 +0,0 @@ -using bts; -using bts.collections; - -using mbts; - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Security.Cryptography; -using BCG; - -namespace bts.BlueOrb { - internal class NonIncJoinBookkeeper : IDisposable { - public Action on_new_data; - private readonly NonIncJoinSpec _join_spec; - private readonly int _join_position; - private readonly int _start_index_in_output; - private readonly BlueRowHasher _row_hasher; - private readonly bool _is_left_most; - private readonly bool _is_tag_along; - private int _n_total_fires = 0, _n_expensive_fires = 0; - - private IBlueTableSchema _schema = null; - private IDictionary _row_map = new Dictionary(); - private List> _row_list = new List>(); - private IBlueRow _solitary_row = null; - - private int[] _join_col_indices = null, - _data_col_indices = null; - - public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) { - _join_spec = the_join_spec; - _join_position = join_position; - _start_index_in_output = start_index_in_output; - _row_hasher = row_hasher; - _is_left_most = _join_position == 0; - _is_tag_along = _join_spec.join_col_names.Length == 0; - } - - private bool _filled_missing_data = false; - private void _fill_missing_data(DataPacket dp) { - if (_filled_missing_data) - return; - Func null_or_empty = (str_arr) => { - if (str_arr == null || str_arr.Length == 0) - return true; - foreach (string s in str_arr) { - if (!String.IsNullOrEmpty(s)) { - return false; - } - } - return true; - }; - Func remove_bad_cols = (col_names) => { - HashSet schema_col_map = new HashSet(dp.schema.columnNames); - List cols_list = new List(col_names); - cols_list.RemoveWhere(c => !schema_col_map.Contains(c)); - return cols_list.ToArray(); - }; - if (null_or_empty(_join_spec.data_col_names)) { - _join_spec.data_col_names = dp.schema.columnNames; - } - _join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names); - if (null_or_empty(_join_spec.data_col_names)) { - _join_spec.data_col_names = dp.schema.columnNames; - } - if (null_or_empty(_join_spec.projected_col_names)) { - _join_spec.projected_col_names = _join_spec.data_col_names; - } - } - - public void init() { - _join_spec.stream.onChange += _on_stream_cb; - } - - public void _on_stream_cb(object dp_obj) { - //Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}"); - var timer = Stopwatch.StartNew(); - DataPacket dp = (DataPacket)dp_obj; - _fill_missing_data(dp); - if (_schema == null) { - _schema = dp.schema; - _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names); - _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names); - } - - if (_is_left_most) { - _row_list.Clear(); - foreach (IBlueRow r in dp.newRows) - _row_list.Add(new Pair(_row_hasher.get_hash(r, _join_col_indices), r)); - } else { - if (_is_tag_along) - _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0); - else { - _row_map.Clear(); - foreach (IBlueRow r in dp.newRows) - _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r; - } - } - timer.Stop(); - _n_total_fires++; - if (timer.ElapsedMilliseconds >= 50) { - _n_expensive_fires++; - Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} " - + $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}"); - } - if (on_new_data != null) - on_new_data(); - } - - public bool is_ready { get { return _schema != null; } } - - public void constribute_cols_to_schema(List cis) { - for (int i = 0; i < _join_spec.data_col_names.Length; i++) - cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType)); - } - - public List> get_list() { return _row_list; } - - public void copy_master_row_to_output_row(int row_index, object[] output_row) { - IBlueRow this_row = _row_list[row_index].second; - BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output); - } - - public void copy_matching_row_to_output_row(int row_hash, object[] output_row) { - if (_is_tag_along) { - if (_solitary_row != null) - BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output); - else - copy_dbnull_to_output_row(output_row); - } else { - IBlueRow this_row = null; - if (_row_map.TryGetValue(row_hash, out this_row)) - BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output); - else - copy_dbnull_to_output_row(output_row); - } - } - - private void copy_dbnull_to_output_row(object[] output_row) { - for (int i = 0; i < _data_col_indices.Length; i++) - output_row[i + _start_index_in_output] = DBNull.Value; - } - - #region IDisposable - - public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; } - - #endregion - } -} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinSpec.cs b/src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinSpec.cs deleted file mode 100644 index 616cab24a..000000000 --- a/src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinSpec.cs +++ /dev/null @@ -1,67 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -using BCG; - -using mbts; - -namespace bts.BlueOrb { - // this a collection of spec, theres no joining logic in here - public class NonIncJoinSpec { - public ViewStream stream; - // join col names are the names on which the streams are joined (for equality), - // data col names are the names in the original stream - // projected col names are their new names, ie if its x:l_x, then x is in the - // data_col_names and l_x in projected cols - public string[] join_col_names, data_col_names, projected_col_names; - - public NonIncJoinSpec() { } - - public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names) - : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { } - - public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) { - stream = the_stream; - join_col_names = the_join_col_names; - data_col_names = the_data_col_names; - projected_col_names = the_projected_col_names; - } - - public static NonIncJoinSpec[] from_triplet(params Triplet[] join_specs) { - List rv = new List(); - foreach (Triplet r in join_specs) { - string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' '); - string[] visible_cols = r.third.Split(' '); - List data_col_names = new List(), mapped_col_names = new List(); - foreach (string s in visible_cols) { - string[] toks = s.Split(':'); - if (toks.Length == 1) { - data_col_names.Add(toks[0]); - mapped_col_names.Add(toks[0]); - } else if (toks.Length == 2) { - data_col_names.Add(toks[0]); - mapped_col_names.Add(toks[1]); - } - } - rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray())); - } - return rv.ToArray(); - } - - public override string ToString() { - Func arr_txt = (a) => { - string rv = ""; - foreach (string s in a) { - rv += "{s} "; - } - rv = rv.Trim(); - return rv; - }; - string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}"; - return rv; - } - } -} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinStream.cs deleted file mode 100644 index 3796711a3..000000000 --- a/src/DataStreamSuite/DataStream/BlueOrb/Join/NonIncJoinStream.cs +++ /dev/null @@ -1,165 +0,0 @@ -using bts; - -using mbts; - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using BCG; - -namespace bts.BlueOrb { - [DataStreamUserRespresentation("JoinStream")] - public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker { - private readonly NonIncJoinSpec[] _join_specs; - private readonly INonIncJoinInlineMapper _output_mapper; - private readonly BlueRowHasher _row_hasher; - private readonly TimeSpan _throttle_period; - private readonly ViewStreamDatabase _vsd; - - private List _book_keeping_data = new List(); - private IBlueTableSchema _output_schema = null; - private DateTime _last_fire_time = DateTime.MinValue; - private bool _is_dirty = false; - - public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj, - ViewStream right, string right_join_cols, string right_proj) - : this(new Triplet[] - { - new Triplet (left, left_join_cols, left_proj), - new Triplet (right, right_join_cols, right_proj) - }) { } - - public NonIncJoinStream(params Triplet[] join_specs) - : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { } - - public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms, - params Triplet[] join_specs) - : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { - } - - public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) - : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { } - - public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) { - _join_specs = join_specs; - _output_mapper = output_mapper; - _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length); - _throttle_period = throttles_ms == 0 ? TimeSpan.Zero : new TimeSpan(0, 0, 0, 0, throttles_ms); - _vsd = vsd; - } - - protected override void myOnFirstSubscriber() { - int data_col_no = 1; - for (int jp = 0; jp < _join_specs.Length; jp++) { - NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher); - //Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}"); - data_col_no += _join_specs[jp].data_col_names.Length; - s.on_new_data += _stream_callback; - _book_keeping_data.Add(s); - } - if (_output_mapper != null) - _output_mapper.onMapperChange += _on_mapper_change; - _vsd.addWorker(this); - foreach (var bkd in _book_keeping_data) - bkd.init(); - } - - protected override void myOnLastSubscriber() { - foreach (NonIncJoinBookkeeper d in _book_keeping_data) - d.Dispose(); - if (_output_mapper != null) - _output_mapper.onMapperChange -= _on_mapper_change; - _vsd.removeWorker(this); - } - - private void _stream_callback() { - //Log.Gen.info("NonIncJoinStream._stream_callback received callback"); - _is_dirty = true; - doWork(); - } - - private void _on_mapper_change() { - _is_dirty = true; - doWork(); - } - - private bool _is_ready() { - bool rv = true; - foreach (NonIncJoinBookkeeper d in _book_keeping_data) - rv &= d.is_ready; - return rv; - } - - private void _construct_schema() { - if (_output_schema == null) { - List cis = new List(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }); - foreach (NonIncJoinBookkeeper d in _book_keeping_data) - d.constribute_cols_to_schema(cis); - _output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray()); - } - } - - private int _n_total_fires = 0, _n_expensive_fires = 0; - private void _form_and_fire() { - //Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}"); - if (_is_ready()) { - _construct_schema(); - var timer = Stopwatch.StartNew(); - IList> left_row_list = _book_keeping_data[0].get_list(); - List output_rows = new List(); - - for (int i = 0; i < left_row_list.Count; i++) { - object[] or = new object[_output_schema.columnInfo.Length]; - or[0] = i; - output_rows.Add(or); - } - - for (int ji = 0; ji < _book_keeping_data.Count; ji++) { - NonIncJoinBookkeeper bk_data = _book_keeping_data[ji]; - for (int i = 0; i < left_row_list.Count; i++) { - if (ji == 0) - bk_data.copy_master_row_to_output_row(i, output_rows[i]); - else - bk_data.copy_matching_row_to_output_row(left_row_list[i].first, output_rows[i]); - } - } - IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows)); - timer.Stop(); - _n_total_fires++; - //if (timer.ElapsedMilliseconds >= 250) { - _n_expensive_fires++; - //string log_msg = $"Join and map time for {_book_keeping_data.Count} " - // + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}"; - //Log.Gen.info(log_msg); - //} - var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true); - myFireEvent(dp_out); - //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows)); - } - } - - private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) { - if (_output_mapper != null) { - IBlueRowCollection output_rows; - var timer = Stopwatch.StartNew(); - output_rows = _output_mapper.map_rows(unmapped_rows); - timer.Stop(); - if (timer.ElapsedMilliseconds >= 50) { - Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}"); - } - return output_rows; - } else { - return unmapped_rows; - } - } - - public void doWork() { - //Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}"); - if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) { - _is_dirty = false; - _last_fire_time = DateTime.Now; - _form_and_fire(); - } - } - } -} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs new file mode 100644 index 000000000..42286727d --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs @@ -0,0 +1,187 @@ +namespace bts.BlueOrb { + using BCG; + using RowHash = System.Int32; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + using System.Collections; + using System.Data.Entity.Core.Mapping; + + [DataStreamUserRespresentation("JoinStream")] + public class SimpleJoinStream : ViewStream { + public class JoinBookkeepingData { + // initialized in cons + private readonly List _join_col_names; + private readonly bool _is_left; + private readonly BlueRowHasher _hasher; + // initialized on first udpate + private IBlueTableSchema _schema; + private int[] _join_cols; + private IDictionary> _hash_row_map = new Dictionary>(); + + public JoinBookkeepingData(ViewStream vs, List join_cols, bool is_left, BlueRowHasher row_hasher) { + _join_col_names = join_cols.Where (c => !String.IsNullOrEmpty(c)).ToList (); + _is_left = is_left; + _hasher = row_hasher; + } + + public IBlueTableSchema schema => _schema; + + public IDictionary> rows => _hash_row_map; + + public List get_matching_rows (int hash) { + List rv = null; + _hash_row_map.TryGetValue(hash, out rv); + return rv; + } + + private RowHash _get_hash(IBlueRow r) { + if (_join_cols == null || _join_cols.Length == 0) { + return 0; + } + return _hasher.get_hash(r, _join_cols); + } + + public void update_data(DataPacket dp) { + if (!ready) { + _schema = dp.newRows.schema; + _join_cols = _join_col_names.Count == 0 ? new int[] { }: _join_col_names.Select(n => dp.schema[n].colNo).ToArray(); + } + _hash_row_map.Clear(); + foreach (IBlueRow r in dp.newRows) { + int hash = _get_hash(r); + List hash_rows = null; + if (!_hash_row_map.TryGetValue(hash, out hash_rows)) { + hash_rows = new List(); + _hash_row_map.Add(hash, hash_rows); + } + hash_rows.Add(r); + } + } + + public bool ready { + get { + return _schema != null; + } + } + + } + + private ViewStream _ls, _rs; + private JoinBookkeepingData _ldata, _rdata; + private ViewStreamDatabase _vsd; + private IBlueTableSchema _schema = null; + + public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) : + this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { } + + public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) { + _ls = ls; + _rs = rs; + List lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList(); + List rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList(); + Log.assert(lj_col_list.Count == rj_col_list.Count); + int n_cols = lj_col_list.Count; + BlueRowHasher hasher = new BlueRowHasher(n_cols); + _ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher); + _rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher); + } + + protected override void myOnFirstSubscriber() { + _ls.onChange += _ls_onchange; + _rs.onChange += _rs_onchange; + } + + private void _ls_onchange(object dp_obj) { + DataPacket dp = dp_obj as DataPacket; + _ldata.update_data(dp); + _form_and_fire(); + } + + private void _rs_onchange(object dp_obj) { + DataPacket dp = dp_obj as DataPacket ; + _rdata.update_data(dp); + _form_and_fire(); + } + + protected override void myOnLastSubscriber() { + _ls.onChange -= _ls_onchange; + _rs.onChange -= _rs_onchange; + } + + private void _build_schema() { + if (_schema != null) + return; + List cis = new List(); + cis.Add(new ColumnInfo("id", 0, TypeCode.Int32)); + foreach (ColumnInfo ci in _ldata.schema.columnInfo) { + string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName; + cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType)); + } + foreach (ColumnInfo ci in _rdata.schema.columnInfo) { + string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName; + cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType)); + } + _schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray()); + } + + private void _form_and_fire() { + if (!_ldata.ready||!_rdata.ready) + return; + _build_schema(); + List res_rows = new List(); + + Action copy_row = (rval, r, is_left) => { + int offset = 1; + JoinBookkeepingData sdata = _ldata; + if (is_left) { + offset = 1; + sdata = _ldata; + } else { + sdata = _rdata; + offset = 1 + _ldata.schema.columnInfo.Length; + } + for (int i = 0; i < sdata.schema.columnInfo.Length; i++) { + rval[offset + i] = r[i]; + } + }; + Func build_row_array = (id, l_row) => { + object[] rv = new object[_schema.columnNames.Length]; + for (int i = 0; i < _schema.columnNames.Length; i++) { + rv[i] = DBNull.Value; + } + rv[0] = id; + copy_row(rv, l_row, true); + return rv; + }; + // this is where the join happens + foreach (var kv in _ldata.rows) { + RowHash hash = kv.Key; + List l_rows = kv.Value; + List r_rows = _rdata.get_matching_rows(hash); + + foreach (IBlueRow lr in l_rows) { + if (r_rows != null) { + foreach (IBlueRow r in r_rows) { + object[] rowvals = build_row_array(res_rows.Count, lr); + copy_row(rowvals, r, false); + res_rows.Add(rowvals); + } + } else { + object[] rowvals = build_row_array(res_rows.Count, lr); + res_rows.Add(rowvals); + } + } + } + BlueTable rv = new BlueTable(_schema); + foreach (object[] rval in res_rows) { + rv.addRow(new BlueRow(_schema, rval)); + } + var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket); + myFireEvent(dp); + } + + } +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs index dabcd7a7c..4446ccc6a 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs @@ -49,14 +49,14 @@ namespace bts.BlueOrb { } } } - public class MSecondStream : TimeUnitStream { + internal class MSecondStream : TimeUnitStream { public MSecondStream(int intervalInms, int size) : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { } protected override DateTime myRoundTime(DateTime dt) { return dt; } } - public class DisplayableMillisecondStream : ViewStream, IStreamMapper { + internal class DisplayableMillisecondStream : ViewStream, IStreamMapper { private IBlueTableSchema mySchema = new BlueTableSchema("ms", new ColumnInfo("id", 0, typeof(int)), new ColumnInfo("time", 1, typeof(string))); diff --git a/src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs b/src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs index c8d07b356..a4b03389e 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs @@ -20,13 +20,14 @@ namespace bts.BlueOrb { Sum, EdgeDiff, EdgeDiffPercentage, - //IsEdgeDiff, } public class MultiRowOpHelper { + // input + op public readonly List> operations; public static MultiRowOpHelper parse_ops(string text) { // text is inp_col: op, inp_col: op, inp_col:op + // ex: "open:First, close:Last, high:Max, low:Min, volume:Sum") List> rv = new List>(); string[] calcs = text.Split(','); foreach (string c in calcs) { @@ -71,6 +72,35 @@ namespace bts.BlueOrb { return or; } + // aggregates multiple rows into one, used by MultiRowRollingMathStream + public static IBlueRow group_by_compute(List rows, int start_idx, int n_rows, MultiRowOpHelper parameters) { + if (rows.Count() == 0) { + return null; + } + IBlueRow ir = rows.ElementAt(start_idx); + IBlueTableSchema in_schema = ir.schema; + IBlueTableSchema out_schema = in_schema; + // out_col vs inp_cal + type + IDictionary> out_col_map = + new Dictionary>(); + foreach (var t in parameters.operations) { + string out_col_name = $"{t.Item1}_{t.Item2}".ToLower(); + out_schema = out_schema.add_column(TypeCode.Double, out_col_name); + out_col_map[out_col_name] = t; + } + MutableBlueRow or = new MutableBlueRow(out_schema, ir); + foreach (var kv in out_col_map) { + string out_col_name = kv.Key; + string in_col_name = kv.Value.Item1; + MultiRowOp op = kv.Value.Item2; + IEnumerable col_vals = BlueDataUtility.get_colvalues(rows, start_idx, n_rows, in_col_name); + double comp_res = compute_op(col_vals, op); + or.setValue(out_col_name, comp_res); + or.setValue("id", ir["id"]); + } + return or; + } + // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations. @@ -107,9 +137,13 @@ namespace bts.BlueOrb { case MultiRowOp.Min: return col_vals.Min(); case MultiRowOp.Mean: - return col_vals.Sum() / col_vals.Count(); + return MathNet.Numerics.Statistics.Statistics.Mean(col_vals); case MultiRowOp.Sum: return col_vals.Sum(); + case MultiRowOp.Median: + return MathNet.Numerics.Statistics.Statistics.Median(col_vals); + //case MultiRowOp.Mode: + // return MathNet.Numerics.Statistics.Statistics.(col_vals); case MultiRowOp.First: return col_vals.First(); case MultiRowOp.Last: diff --git a/src/DataStreamSuite/DataStream/BlueOrb/MultiRowRollingMathStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/MultiRowRollingMathStream.cs new file mode 100644 index 000000000..5d12cbe68 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/MultiRowRollingMathStream.cs @@ -0,0 +1,40 @@ +namespace bts.BlueOrb { + using BCG; + + using System.Collections.Generic; + using System.Net; + + [DataStreamUserRespresentation("MultiRowRollingMathStream")] + public class MultiRowRollingMathStream : ViewStream { + private ViewStream _sorted_parent_str; + private MultiRowOpHelper _op; + private int _rolling_window_sz; + public MultiRowRollingMathStream(ViewStream parent_str, string op_txt, string sort_exp, int rolling_window_sz) { + _sorted_parent_str = new SortStream(parent_str, sort_exp); + _op = MultiRowOpHelper.parse_ops(op_txt); + _rolling_window_sz = rolling_window_sz; + } + + protected override void myOnFirstSubscriber() { + _sorted_parent_str.onChange += _parent_str_onchange; + } + + protected override void myOnLastSubscriber() { + _sorted_parent_str.onChange -= _parent_str_onchange; + } + + private void _parent_str_onchange(object dp_obj) { + DataPacket dp = dp_obj as DataPacket; + var rows = dp.newRows.to_list(); + BlueTable out_table = null; + for (int i = 0; i < rows.Count; i++) { + var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op); + if (out_table == null) + out_table = new BlueTable(out_r.schema); + out_table.addRow(out_r); + } + var dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket, true); + myFireEvent(dp_out); + } + } +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs index eb65d37df..1e23ea149 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs @@ -1,8 +1,5 @@ namespace bts.BlueOrb { using BCG; - - using ProtoBuf; - using System; using System.Collections; using System.Collections.Generic; @@ -26,7 +23,7 @@ namespace bts.BlueOrb { #endregion -#region Row with mapipings build in +#region Row with mappings build in internal class NewProjectRow : IBlueRow { #region IBlueRow members private ProjectData _project_data; @@ -135,38 +132,30 @@ namespace bts.BlueOrb { return new NewProjectRow(_project_data, row); } + // this is where most of the logic is, in terms of translating cols private static ProjectData _create_project_data(IBlueTableSchema input_schema, string project_expression) { project_expression = project_expression.Trim(); - string[] project_toks = project_expression.Split(' '); + string[] project_toks = project_expression.Split(' ', ','); HashSet removed_cols = new HashSet(); - Dictionary mapped_to_orig_map = new Dictionary(); - Dictionary orig_to_mapped = new Dictionary(); + Dictionary orig_to_mapped = new Dictionary(); // by default every col will show up, so if we have the projection as (a:a1) // it will convert a to a1, + // modifications (6/30/2025): + // only the columns represented here, either mapped + // or not, will show // examples: if a schema has cols a b c - // project(parent_str, "a:aa ") -> aa b c - // project(parent_str, "a:aa !c") -> aa b - // project(parent_str, "a:aa !") -> aa - // project(parent_str, "a:aa b")) -> aa b - bool hide_all = false; - foreach (string p_tok in project_toks) { - if (p_tok.StartsWith("!")) { - if (p_tok.Length == 1) { - // if it is a "blanket "!"=> hide all cols except the one that - // are stated explicitly - hide_all = true; - } else { - string removed_col = p_tok.Substring(1); - removed_cols.Add(removed_col); - } - } else if (p_tok.Contains(":")) { + // project(parent_str, "a:aa b") -> aa b + foreach (string o_p_tok in project_toks) { + string p_tok = o_p_tok.Trim(); + if (p_tok.Contains(":")) { + // explicit mapping string[] map_toks = p_tok.Split(":"); string orig_col = map_toks[0]; string mapped_col = map_toks[1]; - mapped_to_orig_map.Add(mapped_col, orig_col); - orig_to_mapped.Add(orig_col, mapped_col); + orig_to_mapped.Add(orig_col, mapped_col); } else { + // this is a col_name explicitely stated if (input_schema.columnNames.Contains(p_tok)) { string orig_col = p_tok; string mapped_col = orig_col; @@ -181,27 +170,19 @@ namespace bts.BlueOrb { if (ci.colName == "id") { ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType); new_cols.Add(nc); - continue; - } - if (removed_cols.Contains(ci.colName)) { - continue; + continue; } else if (orig_to_mapped.ContainsKey(ci.colName)) { string mapped_name = orig_to_mapped[ci.colName]; ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType); new_cols.Add(nc); - } else { - if (hide_all) - continue; - //if (ci.colName == "id" || removed_cols.Count > 0) { - // unless columns are explicitely hidden, all columns show up - ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType); - new_cols.Add(nc); - //} } } IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray()); + var mapped_to_orig_map = orig_to_mapped.ToDictionary(kv => kv.Value, kv => kv.Key); return new ProjectData(input_schema, new_schema, mapped_to_orig_map); } + + } } diff --git a/src/DataStreamSuite/DataStream/BlueOrb/PythonDataframeStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/PythonDataframeStream.cs new file mode 100644 index 000000000..c3d784af0 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/PythonDataframeStream.cs @@ -0,0 +1,92 @@ +namespace bts.BlueOrb { + using BCG; + using System.IO; + using System.Diagnostics; + using System; + + [DataStreamUserRespresentation("PythonDataframeStream")] + public class PythonDataframeStream : ViewStream { + private ViewStream _parent_str; + private readonly string _parentstream_csv_file_name; + public PythonDataframeStream(ViewStream parent_str) { + string id = System.Guid.NewGuid(); + _parentstream_csv_file_name = $"pdfs_in_{id}"; + _parent_str = new CSVSaverStream(parent_str, _parentstream_csv_file_name); + //_simple_test(); + } + + protected override void myOnFirstSubscriber() { + _parent_str.onChange += _parent_str_onchange; + } + + protected override void myOnLastSubscriber() { + _parent_str.onChange -= _parent_str_onchange; + } + + private void _parent_str_onchange(object) { + throw new NotImplementedException(); + + } + + private void _compute() { + string py_func_wrapper_main = @" +import sys +from user_code import test + +if __name__ == '__main__': + inp_file_name = sys.argv[1] + output_file_name = sys.argv[2] +# 1. load inp_file_name into df +# 2. do some test calc +# 3. write the output of 2 into output_file_name + rv = test(arg, out_path) + rv = rv + 12 + with open(out_path, 'w') as f: + f.write(str(rv))"; + } + + private void _simple_test() { + + // actual function that needs to be called //////////////////////// + string py_real_function = @" +def test(x, out_path): + rv = x * 10 + with open(out_path, 'w') as f: + f.write(str(rv)) + return rv"; + // the function is in a string object. + File.WriteAllText("user_code.py", py_real_function); + + + // runner app (top level main function) ////////////////////////////// + string py_func_wrapper_main = @" +import sys +from user_code import test + +if __name__ == '__main__': + arg = int(sys.argv[1]) + out_path = sys.argv[2] + rv = test(arg, out_path) + rv = rv + 12 + with open(out_path, 'w') as f: + f.write(str(rv))"; + + File.WriteAllText("runner.py", py_func_wrapper_main); + + string outputPath = "py_out.txt"; + ProcessStartInfo psi = new ProcessStartInfo { + FileName = "python", + Arguments = $"runner.py 4 {outputPath}", + RedirectStandardOutput = false, + UseShellExecute = false + }; + var proc = Process.Start(psi); + proc.WaitForExit(); + + // Now read the output written by the function + string result = File.ReadAllText(outputPath); + Console.WriteLine(result); // Should print "6" + + } + } +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/SortStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/SortStream.cs new file mode 100644 index 000000000..301728988 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/SortStream.cs @@ -0,0 +1,17 @@ +namespace bts.BlueOrb { + using BCG; + + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + + [DataStreamUserRespresentation("SortStream")] + public class SortStream: ViewStream { + public SortStream (ViewStream parent_stream, string sort_expr) { + ViewStream rs = new BCG.SortStream(parent_stream, sort_expr); + mySetRealSource(rs); + } + } +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/TimeBasedRollingFilterStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/TimeBasedRollingFilterStream.cs new file mode 100644 index 000000000..c3181f103 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/TimeBasedRollingFilterStream.cs @@ -0,0 +1,16 @@ +namespace bts.BlueOrb { + using BCG; + + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + [DataStreamUserRespresentation("TimeBasedRollingFilterStream")] + public class TimeBasedRollingFilterStream : ViewStream { + public TimeBasedRollingFilterStream(ViewStream parent_str, string time_col, double n_seconds) { + ViewStream rs = new BCG.TimeBasedFilterStream(ViewStreamDatabase.instance, parent_str, time_col, TimeSpan.FromSeconds(n_seconds)); + mySetRealSource(rs); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs similarity index 98% rename from src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs rename to src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs index 1b25740fd..3e308c76e 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/TimeSeriesStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs @@ -2,7 +2,7 @@ using System; using BCG; -namespace bts.SilverPine { +namespace bts.BlueOrb { // Keeps adding changing rows to the stream, hence preserving the history // and creating a time series, for example this can be used to make bbo updates // into a time series of prices/qty diff --git a/src/DataStreamSuite/DataStream/BlueOrb/UnionStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/UnionStream.cs index 9082d7d2f..2c1351733 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/UnionStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/UnionStream.cs @@ -1,261 +1,16 @@ -using System; -using System.Collections.Generic; - -using mbts; -using BCG; - namespace bts.BlueOrb { - internal class MutableAppendTable : IBlueRowCollection { - private class MyRow : IBlueRow { - private IBlueRow myRow; - private int myID; + using System; + using System.Collections.Generic; - public MyRow(IBlueRow r, int id) { - myRow = r; - myID = id; - } + using mbts; + using BCG; - #region IBlueRow Members - - public IBlueTableSchema schema { get { return myRow.schema; } } - - public object this[int index] { - get { - if (myRow.schema[index].colName == "id") - return myID; - else - return myRow[index]; - } - } - - public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } } - - #endregion - - #region IDictionary Members - - public void Add(object key, object value) { throw new Exception("Row is read only."); } - - public void Clear() { throw new Exception("Row is read only."); } - - public bool Contains(object key) { return myRow.Contains(key); } - - public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } - - public bool IsFixedSize { get { return true; } } - - public bool IsReadOnly { get { return true; } } - - public System.Collections.ICollection Keys { get { return myRow.Keys; } } - - public void Remove(object key) { throw new Exception("The method or operation is not implemented."); } - - public System.Collections.ICollection Values { - get { - object[] vals = new object[myRow.Count]; - CopyTo(vals, 0); - return vals; - } - } - - public object this[object key] { - get { return key is string ? this[key as string] : this[(int)key]; } - set { throw new Exception("The method or operation is not implemented."); } - } - - #endregion - - #region ICollection Members - - public void CopyTo(Array array, int index) { - myRow.CopyTo(array, index); - array.SetValue(myID, index + myRow.schema["id"].colNo); - } - - public int Count { get { return myRow.Count; } } - - public bool IsSynchronized { get { return myRow.IsSynchronized; } } - - public object SyncRoot { get { return myRow.SyncRoot; } } - - #endregion - - #region IEnumerable Members - - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); } - - #endregion - } - - private BlueTable myRows; - private int myID = 0; - /// - /// maps the table no to a dictionary containing the - /// local IDs to a row in the final big table - /// - private Dictionary> myIDMap = new Dictionary>(); - - public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); } - - public IBlueRow addRow(IBlueRow r, int tableNo) { - if (!myIDMap.ContainsKey(tableNo)) - myIDMap.Add(tableNo, new Dictionary()); - IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id"); - myRows.addRow(nr); - myIDMap[tableNo].Add((int)r["id"], nr); - return nr; - } - - public IBlueRow removeRow(IBlueRow r, int tableNo) { - IBlueRow dr = myIDMap[tableNo][(int)r["id"]]; - myRows.deleteRow(dr); - myIDMap[tableNo].Remove((int)r["id"]); - return dr; - } - - public IBlueRowCollection removeTable(int tableNo) { - if (!myIDMap.ContainsKey(tableNo)) - return null; - else { - BlueTable rv = new BlueTable(myRows.schema); - foreach (IBlueRow r in myIDMap[tableNo].Values) - rv.addRow(r); - myRows.deleteRows(rv); - myIDMap.Remove(tableNo); - return rv; - } - } - - public void clear() { - myRows.clear(); - myIDMap.Clear(); - } - - #region IBlueRowCollection Members - - public IBlueRow this[int id] { get { return myRows[id]; } } - - public IBlueTableSchema schema { get { return myRows.schema; } } - - public bool containsID(int id) { return myRows.containsID(id); } - - #endregion - - #region ICollection Members - - public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); } - - public int Count { get { return myRows.Count; } } - - public bool IsSynchronized { get { return myRows.IsSynchronized; } } - - public object SyncRoot { get { return myRows.SyncRoot; } } - - #endregion - - #region IEnumerable Members - - public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); } - - #endregion - } - /// - /// MutableStream can append multiple streams - /// as well as give the user the flexibility - /// to append (or remove) more streams at - /// runtime - /// - } -namespace bts.BlueOrb { [DataStreamUserRespresentation("UnionStream")] - public class MutableAppendStream : ViewStream { - private bool mySubscriptionActive = false; - private MutableAppendTable myTable; - private int myMovingIndex; - private Dictionary> myStreamIndex = new Dictionary>(); - private bool _allign_col_orders = true; - - public MutableAppendStream(params ViewStream[] streams) - : this(new List(streams)) { } - - public MutableAppendStream(bool allign_col_order, params ViewStream[] streams) - : this(allign_col_order, new List(streams)) { } - - public MutableAppendStream(ICollection streams) - : this(true, streams) { } - - public MutableAppendStream(bool allign_col_orders, ICollection streams) { - _allign_col_orders = allign_col_orders; - foreach (ViewStream vs in streams) { - myMovingIndex++; - ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString()); - if (_allign_col_orders) - pj = new ProjectStream(pj, "!"); - myStreamIndex.Add(myMovingIndex, new Pair(vs, pj)); - } - } - - public void addStream(ViewStream vs) { - myMovingIndex++; - ///the change name is done to identify the callbacks, and the project to - ///make all the schemas col orders the same - ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString()); - if (_allign_col_orders) - pj = new ProjectStream(pj, "!"); - myStreamIndex.Add(myMovingIndex, new Pair(vs, pj)); - if (mySubscriptionActive) - pj.onChange += new ChangeDelegate(myHandleStreamChanges); - } - - public void removeStream(ViewStream vs) { - int index = -1; - foreach (KeyValuePair> p in myStreamIndex) { - if (p.Value.first == vs) { - index = p.Key; - if (mySubscriptionActive) { - p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges); - if (myTable != null) { - IBlueRowCollection delRows = myTable.removeTable(p.Key); - if (delRows != null) - myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows)); - } - } - break; - } - if (index > 0) - myStreamIndex.Remove(index); - } - } - - protected override void myOnFirstSubscriber() { - mySubscriptionActive = true; - foreach (KeyValuePair> p in myStreamIndex) - p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges); - } - - protected override void myOnLastSubscriber() { - mySubscriptionActive = false; - foreach (KeyValuePair> p in myStreamIndex) - p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges); - if (myTable != null) - myTable.clear(); - } - - private void myHandleStreamChanges(object newValue) { - DataPacket dp = newValue as DataPacket; - myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo)); - BlueTable adds = new BlueTable(myTable.schema), - dels = new BlueTable(myTable.schema); - int index = Int32.Parse(dp.schema.tableName); - foreach (IBlueRow r in dp.deletedRows) { - IBlueRow dr = myTable.removeRow(r, index); - dels.addRow(dr); - } - foreach (IBlueRow r in dp.addedRows) { - IBlueRow nr = myTable.addRow(r, index); - adds.addRow(nr); - } - myFireEvent(new DataPacket(myTable, adds, dels)); + public class UnionStream : ViewStream { + public UnionStream(ViewStream left_stream, ViewStream right_stream) { + ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream); + mySetRealSource(rs); } } + } diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs index 321087674..93dfc89f5 100644 --- a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs @@ -1,33 +1,51 @@ using System; +using System.Collections.Generic; -namespace BCG { +namespace BCG { public class AddColumnsStream : ViewStream { - public static Tuple parse_exp(string exp) { + public static Tuple parse_single_tok(string exp) { string[] toks = exp.Split(':'); - string col_name = toks[0]; - string col_value = toks[1]; + string col_name = toks[0].Trim(); + string col_value = toks[1].Trim(); Tuple tup = BlueDataUtility.guess_type(col_value); return Tuple.Create(col_name, tup.Item1, tup.Item2); } + public static List> parse_exp(string exp) { + string[] toks = exp.Split(","); + List> rv = new List>(); + foreach (string s in toks) { + var tup = parse_single_tok(s.Trim()); + rv.Add(tup); + } + return rv; + } + private Tuple _new_col; + private List> _new_col_map; private IBlueTableSchema _out_schema; public AddColumnsStream(ViewStream vs, string col_def) { - _new_col = parse_exp(col_def); + _new_col_map = parse_exp(col_def); ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r)); mySetRealSource(rs); } private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) { - if (_out_schema == null) - _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2); + if (_out_schema == null) { + _out_schema = input_schema; + foreach (var m in _new_col_map) { + _out_schema = BlueDataUtility.add_column(_out_schema, m.Item1, m.Item2); + } + } return _out_schema; } private IBlueRow _map(IBlueRow r) { MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r); - rv.setValue(_new_col.Item1, _new_col.Item3); + foreach (var m in _new_col_map) { + rv.setValue(m.Item1, m.Item3); + } return rv; } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs index 4742545b1..02fc44fc9 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs @@ -52,14 +52,14 @@ namespace BCG { } public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive) - : this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) { + : this(vsd, rowTimeColumnName, rowTimeToLive, rowTimeToLive) { } - public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) { + public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filter_period) { myVsd = vsd; _time_col = rowTimeColumnName; _ttl = rowTimeToLive; - myFilterFrequency = filterFrequency; + _filter_period = filter_period; myNextFilter = DateTime.MinValue; // add worker @@ -92,7 +92,7 @@ namespace BCG { // fire event onFilterChange(); - myNextFilter = now.Add(myFilterFrequency); + myNextFilter = now.Add(_filter_period); } } #endregion @@ -108,7 +108,7 @@ namespace BCG { private ViewStreamDatabase myVsd; private string _time_col; private TimeSpan _ttl; - private TimeSpan myFilterFrequency; + private TimeSpan _filter_period; private DateTime myNextFilter; #endregion } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs index 68e78c382..c3dedbbfe 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs @@ -8,7 +8,7 @@ using bts; using bts.SilverPine; namespace BCG { - [DataStreamUserRespresentation("AggregateStream")] + //[DataStreamUserRespresentation("AggregateStream")] public class NonIncGroupByStream : ViewStream { private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator { Func _groupby_cols_func; diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/FullJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/FullJoinStream.cs index 258345589..def6a397a 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/FullJoinStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/FullJoinStream.cs @@ -1,31 +1,31 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; +//using System; +//using System.Collections.Generic; +//using System.Linq; +//using System.Text; -namespace BCG -{ - public class FullJoinStream : ViewStream - { - private ViewStream myMainStream; - public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1, - ViewStream v2, string joinCol2, string cmExp2, string empty_col) - { - myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2); - ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1); - filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col)); - myMainStream = new AppendStream(myMainStream, filtered); - } +//namespace BCG +//{ +// public class FullJoinStream : ViewStream +// { +// private ViewStream myMainStream; +// public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1, +// ViewStream v2, string joinCol2, string cmExp2, string empty_col) +// { +// myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2); +// ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1); +// filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col)); +// myMainStream = new AppendStream(myMainStream, filtered); +// } - protected override void myOnFirstSubscriber() - { - myMainStream.onChange += myHandleMainStreamChange; - } +// protected override void myOnFirstSubscriber() +// { +// myMainStream.onChange += myHandleMainStreamChange; +// } - private void myHandleMainStreamChange(object newData) - { - IBlueRowCollection result = (newData as DataPacket).newRows; - myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows)); - } - } -} +// private void myHandleMainStreamChange(object newData) +// { +// IBlueRowCollection result = (newData as DataPacket).newRows; +// myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows)); +// } +// } +//} diff --git a/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs index 9300efb1d..c52240096 100644 --- a/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs +++ b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs @@ -12,7 +12,7 @@ using System.Collections.Generic; using System.Data.SQLite; -namespace bts.SilverPine { +namespace bts.BlueOrb{ public static class SQLiteHelper { public static string location = "C:\\Users\\m_tah\\development\\data\\"; diff --git a/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SqliteStream.cs b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SqliteStream.cs index 0dc15e768..8e7fa80e5 100644 --- a/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SqliteStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SqliteStream.cs @@ -1,14 +1,12 @@ -namespace bts.SilverPine { +namespace bts.BlueOrb { using System; using System.Data; using System.Data.SQLite; using System.Diagnostics; using System.Collections; - - using BCG; - [DataStreamUserRespresentation("SQLite")] + [DataStreamUserRespresentation("SQLiteStream")] public class SQLiteStream : ViewStream { private readonly string _connection_string, _query; private Stopwatch _stopwatch = new Stopwatch(); @@ -70,4 +68,5 @@ namespace bts.SilverPine { } -} \ No newline at end of file +} + diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs index e03507ed7..82e745c3c 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs @@ -1,5 +1,6 @@ using System; +using BCG; using BCG.Utilities.General; using bts; @@ -87,3 +88,17 @@ namespace BCG { } } } + +//namespace bts.BlueOrb { +// [DataStreamUserRespresentation("MillisecondStream")] +// public class MillisecondStream : ViewStream { +// public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) { +// ViewStream rs = new BCG.MillisecondStream(vsd, interval_msecs, size); +// mySetRealSource(rs); +// } + +// public MillisecondStream(int interval_msecs, int size) +// : this(interval_msecs, size) { +// } +// } +//} diff --git a/src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs b/src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs index afb888cfa..4fead4136 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs @@ -1,124 +1,124 @@ -namespace bts.SilverPine { - using BCG; +//namespace bts.SilverPine { +// using BCG; - using bts; - using bts.collections; +// using bts; +// //using bts.collections; - using System; - using System.CodeDom; - using System.Collections.Generic; - using System.Linq; +// using System; +// using System.CodeDom; +// using System.Collections.Generic; +// using System.Linq; - // Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows - // so: - // f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2] - [DataStreamUserRespresentation("ColDiff")] - public class ColumnDifferenceStream : ViewStream { - private readonly ViewStream _base_stream; - private readonly FilterStream.OnFilter _f1, _f2; - private readonly string _c1, _c2; +// // Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows +// // so: +// // f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2] +// [DataStreamUserRespresentation("ColDiff")] +// public class ColumnDifferenceStream : ViewStream { +// private readonly ViewStream _base_stream; +// private readonly FilterStream.OnFilter _f1, _f2; +// private readonly string _c1, _c2; - public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) { - _base_stream = base_stream; - _f1 = f1; - _f2 = f2; - _c1 = col1; - _c2 = col2; - } +// public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) { +// _base_stream = base_stream; +// _f1 = f1; +// _f2 = f2; +// _c1 = col1; +// _c2 = col2; +// } - public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) { - FilterStream.OnFilter f1_del = r => { - BooleanExpressionFilter rv = new BooleanExpressionFilter(f1); - return rv.filter(r); - }; - FilterStream.OnFilter f2_del = r => { - BooleanExpressionFilter rv = new BooleanExpressionFilter(f2); - return rv.filter(r); - }; - ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2); - mySetRealSource(rs); - } +// public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) { +// FilterStream.OnFilter f1_del = r => { +// BooleanExpressionFilter rv = new BooleanExpressionFilter(f1); +// return rv.filter(r); +// }; +// FilterStream.OnFilter f2_del = r => { +// BooleanExpressionFilter rv = new BooleanExpressionFilter(f2); +// return rv.filter(r); +// }; +// ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2); +// mySetRealSource(rs); +// } - // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2] - // we should generlize this for any math operation - public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) { - // use GlobalMapStream - string diff_col_perc_name = $"{diff_col_name}_percentage"; - System.Func, uint, BCG.IBlueRow> g_map_func = (rows, idx) => { - int minuend_idx = Convert.ToInt32(idx); - var inp_schema = rows[0].schema; - var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name); - out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name); - // minuend - subtrahend - IBlueRow minuend_row = rows.ElementAt(minuend_idx); - MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row); - int subtrahend_index = minuend_idx + row_offset; - double diff_val = Double.NaN, diff_val_perc = Double.NaN; - if (subtrahend_index >= 0) { - try { - IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index); - diff_val = minuend_row.get(col1) - subtrahend_row.get(col2); - diff_val_perc = 100.0 * diff_val / subtrahend_row.get(col1); - } catch (Exception ex) { - Log.assert(false, ex.Message); - } - } - rv.setValue(diff_col_name, diff_val); - rv.setValue(diff_col_perc_name, diff_val_perc); - return rv; - }; - ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func); - if (optional_perc_diff_filter != 0) { - rs = new FilterStream(rs, r => { - double diff_val_perc = r.get(diff_col_perc_name); - double abs_diff_val_perc = Math.Abs(diff_val_perc); - return abs_diff_val_perc >= optional_perc_diff_filter; - }); - } - mySetRealSource(rs); - } +// // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2] +// // we should generlize this for any math operation +// public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) { +// // use GlobalMapStream +// string diff_col_perc_name = $"{diff_col_name}_percentage"; +// System.Func, uint, BCG.IBlueRow> g_map_func = (rows, idx) => { +// int minuend_idx = Convert.ToInt32(idx); +// var inp_schema = rows[0].schema; +// var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name); +// out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name); +// // minuend - subtrahend +// IBlueRow minuend_row = rows.ElementAt(minuend_idx); +// MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row); +// int subtrahend_index = minuend_idx + row_offset; +// double diff_val = Double.NaN, diff_val_perc = Double.NaN; +// if (subtrahend_index >= 0) { +// try { +// IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index); +// diff_val = minuend_row.get(col1) - subtrahend_row.get(col2); +// diff_val_perc = 100.0 * diff_val / subtrahend_row.get(col1); +// } catch (Exception ex) { +// Log.assert(false, ex.Message); +// } +// } +// rv.setValue(diff_col_name, diff_val); +// rv.setValue(diff_col_perc_name, diff_val_perc); +// return rv; +// }; +// ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func); +// if (optional_perc_diff_filter != 0) { +// rs = new FilterStream(rs, r => { +// double diff_val_perc = r.get(diff_col_perc_name); +// double abs_diff_val_perc = Math.Abs(diff_val_perc); +// return abs_diff_val_perc >= optional_perc_diff_filter; +// }); +// } +// mySetRealSource(rs); +// } - protected override void myOnFirstSubscriber() { - _base_stream.onChange += _handle_base_stream_update; - } +// protected override void myOnFirstSubscriber() { +// _base_stream.onChange += _handle_base_stream_update; +// } - protected override void myOnLastSubscriber() { - _base_stream.onChange -= _handle_base_stream_update; - } +// protected override void myOnLastSubscriber() { +// _base_stream.onChange -= _handle_base_stream_update; +// } - private void _handle_base_stream_update(object dp_obj) { - // could have been done with filter and cartesian prod, and then math - DataPacket dp = dp_obj as DataPacket; - List rows = dp.newRows.to_list(); - var f1_rows = rows.Where(r => _f1(r)); - //Log.assert(f1_rows.Count() == 1); - var f2_rows = rows.Where(r => _f2(r)); - //Log.assert(f2_rows.Count() == 1); - if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) { - IBlueRow f1_r = f1_rows.ElementAt(0); - IBlueRow f2_r = f2_rows.ElementAt(0); - double diff = f1_r.get(_c1) - f2_r.get(_c2); - IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff")); - BlueTable out_table = new BlueTable(out_schema); - MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r); - out_row.setValue("diff", diff); - out_table.addRow(out_row); - DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket); - myFireEvent(dp_out); - } else { - IBlueRow f1_r = rows.ElementAt(0); - IBlueRow f2_r = rows.ElementAt(0); - double diff = Double.NaN; - IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff")); - BlueTable out_table = new BlueTable(out_schema); - MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r); - out_row.setValue("diff", diff); - out_table.addRow(out_row); - DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket); - myFireEvent(dp_out); - } +// private void _handle_base_stream_update(object dp_obj) { +// // could have been done with filter and cartesian prod, and then math +// DataPacket dp = dp_obj as DataPacket; +// List rows = dp.newRows.to_list(); +// var f1_rows = rows.Where(r => _f1(r)); +// //Log.assert(f1_rows.Count() == 1); +// var f2_rows = rows.Where(r => _f2(r)); +// //Log.assert(f2_rows.Count() == 1); +// if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) { +// IBlueRow f1_r = f1_rows.ElementAt(0); +// IBlueRow f2_r = f2_rows.ElementAt(0); +// double diff = f1_r.get(_c1) - f2_r.get(_c2); +// IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff")); +// BlueTable out_table = new BlueTable(out_schema); +// MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r); +// out_row.setValue("diff", diff); +// out_table.addRow(out_row); +// DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket); +// myFireEvent(dp_out); +// } else { +// IBlueRow f1_r = rows.ElementAt(0); +// IBlueRow f2_r = rows.ElementAt(0); +// double diff = Double.NaN; +// IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff")); +// BlueTable out_table = new BlueTable(out_schema); +// MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r); +// out_row.setValue("diff", diff); +// out_table.addRow(out_row); +// DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket); +// myFireEvent(dp_out); +// } - } - } +// } +// } -} \ No newline at end of file +//} diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/AppendStream.cs b/src/DataStreamSuite/DataStream/Primitives/Union/AppendStream.cs index a2458379f..976d015fd 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Union/AppendStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Union/AppendStream.cs @@ -1,118 +1,118 @@ -using System; -using System.Data; -using System.Collections; -using BCG.Utilities; -using BCG.Utilities.General; -namespace BCG -{ - /// - /// AppendStream appends 2 tables with each other, with a generated id having no bearing to the original id. - /// The column ordering the 2 tables need not be the same - /// - public class AppendStream : UnionStream - { - private bool myUseNewAppend = DataStreamLoadFileConfig.instance.useNewAppend; - public AppendStream(ViewStream v1, ViewStream v2) - { - if (myUseNewAppend) - mySetRealSource(new MutableAppendStream(v1, v2)); - else - { - myLeftVs = v1; - myRightVs = v2; - } - } +//using System; +//using System.Data; +//using System.Collections; +//using BCG.Utilities; +//using BCG.Utilities.General; +//namespace BCG +//{ +// /// +// /// AppendStream appends 2 tables with each other, with a generated id having no bearing to the original id. +// /// The column ordering the 2 tables need not be the same +// /// +// public class AppendStream : UnionStream +// { +// private bool myUseNewAppend = DataStreamLoadFileConfig.instance.useNewAppend; +// public AppendStream(ViewStream v1, ViewStream v2) +// { +// if (myUseNewAppend) +// mySetRealSource(new MutableAppendStream(v1, v2)); +// else +// { +// myLeftVs = v1; +// myRightVs = v2; +// } +// } - public AppendStream (params ViewStream[] vss) - { +// public AppendStream (params ViewStream[] vss) +// { - if (myUseNewAppend) - mySetRealSource(new MutableAppendStream(vss)); - else - { - if (vss.Length == 1) - mySetRealSource(vss[0]); - else if (vss.Length > 1) - { - AppendStream rv = null; - foreach (ViewStream s in vss) - rv = rv == null ? new AppendStream(new ViewStream[] { s }) : new AppendStream(rv, s); - mySetRealSource(rv); - } - else - //don't do anything if they didnt' supply any viewstreams - mySetRealSource(null); - } +// if (myUseNewAppend) +// mySetRealSource(new MutableAppendStream(vss)); +// else +// { +// if (vss.Length == 1) +// mySetRealSource(vss[0]); +// else if (vss.Length > 1) +// { +// AppendStream rv = null; +// foreach (ViewStream s in vss) +// rv = rv == null ? new AppendStream(new ViewStream[] { s }) : new AppendStream(rv, s); +// mySetRealSource(rv); +// } +// else +// //don't do anything if they didnt' supply any viewstreams +// mySetRealSource(null); +// } - } +// } - public override void add (IBlueRowCollection rows, bool isLeft) - { - IDictionary index = isLeft ? myLeftIndex : myRightIndex; - int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap; - foreach (IBlueRow row in rows) - { - UnionRow newRow = new UnionRow (myLastFiredPacket.newRows.schema, row, - myIdCounter ++, myIdIndex, indexToIndexMap); - ((BlueTable)myLastFiredPacket.newRows).addRow (newRow); - index[row["id"]] = newRow; - ((BlueTable)(myLastFiredPacket.addedRows)).addRow (newRow); - } - } +// public override void add (IBlueRowCollection rows, bool isLeft) +// { +// IDictionary index = isLeft ? myLeftIndex : myRightIndex; +// int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap; +// foreach (IBlueRow row in rows) +// { +// UnionRow newRow = new UnionRow (myLastFiredPacket.newRows.schema, row, +// myIdCounter ++, myIdIndex, indexToIndexMap); +// ((BlueTable)myLastFiredPacket.newRows).addRow (newRow); +// index[row["id"]] = newRow; +// ((BlueTable)(myLastFiredPacket.addedRows)).addRow (newRow); +// } +// } - public override void delete (IBlueRowCollection rows, bool isLeft) - { - IDictionary index = isLeft ? myLeftIndex : myRightIndex; - foreach (IBlueRow axedRow in rows) - { - int rowId = (int)axedRow["id"]; - IBlueRow row = (IBlueRow)index[rowId]; - index.Remove (rowId); - ((BlueTable)myLastFiredPacket.newRows).deleteRow (row); - ((BlueTable)(myLastFiredPacket.deletedRows)).addRow (row); - } - } +// public override void delete (IBlueRowCollection rows, bool isLeft) +// { +// IDictionary index = isLeft ? myLeftIndex : myRightIndex; +// foreach (IBlueRow axedRow in rows) +// { +// int rowId = (int)axedRow["id"]; +// IBlueRow row = (IBlueRow)index[rowId]; +// index.Remove (rowId); +// ((BlueTable)myLastFiredPacket.newRows).deleteRow (row); +// ((BlueTable)(myLastFiredPacket.deletedRows)).addRow (row); +// } +// } - public override void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft) - { - IDictionary temp = new Hashtable (); - IDictionary index = isLeft ? myLeftIndex : myRightIndex; - int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap; - IBlueRow corrRow; - foreach (IBlueRow dRow in dels) - { - corrRow = index[dRow["id"]] as IBlueRow; - (myLastFiredPacket.newRows as BlueTable).deleteRow (corrRow); - (myLastFiredPacket.deletedRows as BlueTable).addRow (corrRow); - index.Remove (dRow["id"]); - if (adds.containsID ((int)dRow["id"])) - { - IBlueRow r = adds[(int)dRow["id"]]; - corrRow = new UnionRow (myLastFiredPacket.schema, r, (int)corrRow["id"], myIdIndex, indexToIndexMap); - (myLastFiredPacket.newRows as BlueTable).addRow (corrRow); - (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow); - index.Add (r["id"], corrRow); - temp.Add (r["id"], true); - } - } - foreach (IBlueRow aRow in adds) - { - if (!temp.Contains (aRow["id"])) - { - corrRow = new UnionRow (myLastFiredPacket.schema, aRow, myIdCounter++, myIdIndex, indexToIndexMap); - (myLastFiredPacket.newRows as BlueTable).addRow (corrRow); - (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow); - index.Add (aRow["id"], corrRow); - } - } - } - } +// public override void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft) +// { +// IDictionary temp = new Hashtable (); +// IDictionary index = isLeft ? myLeftIndex : myRightIndex; +// int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap; +// IBlueRow corrRow; +// foreach (IBlueRow dRow in dels) +// { +// corrRow = index[dRow["id"]] as IBlueRow; +// (myLastFiredPacket.newRows as BlueTable).deleteRow (corrRow); +// (myLastFiredPacket.deletedRows as BlueTable).addRow (corrRow); +// index.Remove (dRow["id"]); +// if (adds.containsID ((int)dRow["id"])) +// { +// IBlueRow r = adds[(int)dRow["id"]]; +// corrRow = new UnionRow (myLastFiredPacket.schema, r, (int)corrRow["id"], myIdIndex, indexToIndexMap); +// (myLastFiredPacket.newRows as BlueTable).addRow (corrRow); +// (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow); +// index.Add (r["id"], corrRow); +// temp.Add (r["id"], true); +// } +// } +// foreach (IBlueRow aRow in adds) +// { +// if (!temp.Contains (aRow["id"])) +// { +// corrRow = new UnionRow (myLastFiredPacket.schema, aRow, myIdCounter++, myIdIndex, indexToIndexMap); +// (myLastFiredPacket.newRows as BlueTable).addRow (corrRow); +// (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow); +// index.Add (aRow["id"], corrRow); +// } +// } +// } +// } - public class MergeStream : ViewStream - { - public MergeStream (ViewStream v1, ViewStream v2, string[] mergeCols) - { - mySetRealSource (new DistinctRowStream (new AppendStream (v1, v2), mergeCols)); - } - } -} +// public class MergeStream : ViewStream +// { +// public MergeStream (ViewStream v1, ViewStream v2, string[] mergeCols) +// { +// mySetRealSource (new DistinctRowStream (new AppendStream (v1, v2), mergeCols)); +// } +// } +//} diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs b/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs index ba310f455..71ee4336f 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs @@ -2,170 +2,171 @@ using System; using System.Collections.Generic; using mbts; +//using BCG; namespace BCG { - internal class MutableAppendTable : IBlueRowCollection { - private class MyRow : IBlueRow { - private IBlueRow myRow; - private int myID; - - public MyRow(IBlueRow r, int id) { - myRow = r; - myID = id; - } - - #region IBlueRow Members - - public IBlueTableSchema schema { get { return myRow.schema; } } - - public object this[int index] { - get { - if (myRow.schema[index].colName == "id") - return myID; - else - return myRow[index]; - } - } - - public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } } - - #endregion - - #region IDictionary Members - - public void Add(object key, object value) { throw new Exception("Row is read only."); } - - public void Clear() { throw new Exception("Row is read only."); } - - public bool Contains(object key) { return myRow.Contains(key); } - - public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } - - public bool IsFixedSize { get { return true; } } - - public bool IsReadOnly { get { return true; } } - - public System.Collections.ICollection Keys { get { return myRow.Keys; } } - - public void Remove(object key) { throw new Exception("The method or operation is not implemented."); } - - public System.Collections.ICollection Values { - get { - object[] vals = new object[myRow.Count]; - CopyTo(vals, 0); - return vals; - } - } - - public object this[object key] { - get { return key is string ? this[key as string] : this[(int)key]; } - set { throw new Exception("The method or operation is not implemented."); } - } - - #endregion - - #region ICollection Members - - public void CopyTo(Array array, int index) { - myRow.CopyTo(array, index); - array.SetValue(myID, index + myRow.schema["id"].colNo); - } - - public int Count { get { return myRow.Count; } } - - public bool IsSynchronized { get { return myRow.IsSynchronized; } } - - public object SyncRoot { get { return myRow.SyncRoot; } } - - #endregion - - #region IEnumerable Members - - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); } - - #endregion - } - - private BlueTable myRows; - private int myID = 0; - /// - /// maps the table no to a dictionary containing the - /// local IDs to a row in the final big table - /// - private Dictionary> myIDMap = new Dictionary>(); - - public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); } - - public IBlueRow addRow(IBlueRow r, int tableNo) { - if (!myIDMap.ContainsKey(tableNo)) - myIDMap.Add(tableNo, new Dictionary()); - IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id"); - myRows.addRow(nr); - myIDMap[tableNo].Add((int)r["id"], nr); - return nr; - } - - public IBlueRow removeRow(IBlueRow r, int tableNo) { - IBlueRow dr = myIDMap[tableNo][(int)r["id"]]; - myRows.deleteRow(dr); - myIDMap[tableNo].Remove((int)r["id"]); - return dr; - } - - public IBlueRowCollection removeTable(int tableNo) { - if (!myIDMap.ContainsKey(tableNo)) - return null; - else { - BlueTable rv = new BlueTable(myRows.schema); - foreach (IBlueRow r in myIDMap[tableNo].Values) - rv.addRow(r); - myRows.deleteRows(rv); - myIDMap.Remove(tableNo); - return rv; - } - } - - public void clear() { - myRows.clear(); - myIDMap.Clear(); - } - - #region IBlueRowCollection Members - - public IBlueRow this[int id] { get { return myRows[id]; } } - - public IBlueTableSchema schema { get { return myRows.schema; } } - - public bool containsID(int id) { return myRows.containsID(id); } - - #endregion - - #region ICollection Members - - public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); } - - public int Count { get { return myRows.Count; } } - - public bool IsSynchronized { get { return myRows.IsSynchronized; } } - - public object SyncRoot { get { return myRows.SyncRoot; } } - - #endregion - - #region IEnumerable Members - - public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); } - - #endregion - } - /// + /// /// MutableStream can append multiple streams /// as well as give the user the flexibility /// to append (or remove) more streams at /// runtime /// - [DataStreamUserRespresentation("UnionStream")] + //[DataStreamUserRespresentation("UnionStream")] public class MutableAppendStream : ViewStream { + internal class MutableAppendTable : IBlueRowCollection { + private class MyRow : IBlueRow { + private IBlueRow myRow; + private int myID; + + public MyRow(IBlueRow r, int id) { + myRow = r; + myID = id; + } + + #region IBlueRow Members + + public IBlueTableSchema schema { get { return myRow.schema; } } + + public object this[int index] { + get { + if (myRow.schema[index].colName == "id") + return myID; + else + return myRow[index]; + } + } + + public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } } + + #endregion + + #region IDictionary Members + + public void Add(object key, object value) { throw new Exception("Row is read only."); } + + public void Clear() { throw new Exception("Row is read only."); } + + public bool Contains(object key) { return myRow.Contains(key); } + + public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } + + public bool IsFixedSize { get { return true; } } + + public bool IsReadOnly { get { return true; } } + + public System.Collections.ICollection Keys { get { return myRow.Keys; } } + + public void Remove(object key) { throw new Exception("The method or operation is not implemented."); } + + public System.Collections.ICollection Values { + get { + object[] vals = new object[myRow.Count]; + CopyTo(vals, 0); + return vals; + } + } + + public object this[object key] { + get { return key is string ? this[key as string] : this[(int)key]; } + set { throw new Exception("The method or operation is not implemented."); } + } + + #endregion + + #region ICollection Members + + public void CopyTo(Array array, int index) { + myRow.CopyTo(array, index); + array.SetValue(myID, index + myRow.schema["id"].colNo); + } + + public int Count { get { return myRow.Count; } } + + public bool IsSynchronized { get { return myRow.IsSynchronized; } } + + public object SyncRoot { get { return myRow.SyncRoot; } } + + #endregion + + #region IEnumerable Members + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); } + + #endregion + } + + private BlueTable myRows; + private int myID = 0; + /// + /// maps the table no to a dictionary containing the + /// local IDs to a row in the final big table + /// + private Dictionary> myIDMap = new Dictionary>(); + + public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); } + + public IBlueRow addRow(IBlueRow r, int tableNo) { + if (!myIDMap.ContainsKey(tableNo)) + myIDMap.Add(tableNo, new Dictionary()); + IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id"); + myRows.addRow(nr); + myIDMap[tableNo].Add((int)r["id"], nr); + return nr; + } + + public IBlueRow removeRow(IBlueRow r, int tableNo) { + IBlueRow dr = myIDMap[tableNo][(int)r["id"]]; + myRows.deleteRow(dr); + myIDMap[tableNo].Remove((int)r["id"]); + return dr; + } + + public IBlueRowCollection removeTable(int tableNo) { + if (!myIDMap.ContainsKey(tableNo)) + return null; + else { + BlueTable rv = new BlueTable(myRows.schema); + foreach (IBlueRow r in myIDMap[tableNo].Values) + rv.addRow(r); + myRows.deleteRows(rv); + myIDMap.Remove(tableNo); + return rv; + } + } + + public void clear() { + myRows.clear(); + myIDMap.Clear(); + } + + #region IBlueRowCollection Members + + public IBlueRow this[int id] { get { return myRows[id]; } } + + public IBlueTableSchema schema { get { return myRows.schema; } } + + public bool containsID(int id) { return myRows.containsID(id); } + + #endregion + + #region ICollection Members + + public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); } + + public int Count { get { return myRows.Count; } } + + public bool IsSynchronized { get { return myRows.IsSynchronized; } } + + public object SyncRoot { get { return myRows.SyncRoot; } } + + #endregion + + #region IEnumerable Members + + public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); } + + #endregion + } private bool mySubscriptionActive = false; private MutableAppendTable myTable; private int myMovingIndex; @@ -173,7 +174,8 @@ namespace BCG { private bool _allign_col_orders = true; public MutableAppendStream(params ViewStream[] streams) - : this(new List(streams)) { } + : this(new List(streams)) { + } public MutableAppendStream(bool allign_col_order, params ViewStream[] streams) : this(allign_col_order, new List(streams)) { } @@ -256,3 +258,13 @@ namespace BCG { } } } + + +//namespace bts.BlueOrb { +// public class UnionStream : ViewStream { +// public UnionStream(ViewStream left_stream, ViewStream right_stream) { +// ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream); +// mySetRealSource(rs); +// } +// } +//} diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/UnionOperand.cs b/src/DataStreamSuite/DataStream/Primitives/Union/UnionOperand.cs index 08c0effb3..d5e32d55e 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Union/UnionOperand.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Union/UnionOperand.cs @@ -1,63 +1,63 @@ -using System; +//using System; -namespace BCG -{ - /// - /// Respresents a Union Operand. Hooks up to the Stream delegate, and alerts the UnionStream - /// on receiving updates - /// - public class UnionOperand - { - private bool myIsLeft; - private DataPacket myLastArrivedPacket; - private UnionStream myMaster; +//namespace BCG +//{ +// /// +// /// Respresents a Union Operand. Hooks up to the Stream delegate, and alerts the UnionStream +// /// on receiving updates +// /// +// public class UnionOperand +// { +// private bool myIsLeft; +// private DataPacket myLastArrivedPacket; +// private UnionStream myMaster; - public UnionOperand (bool isLeft, UnionStream master) - { - myIsLeft = isLeft; - myMaster = master; - } +// public UnionOperand (bool isLeft, UnionStream master) +// { +// myIsLeft = isLeft; +// myMaster = master; +// } - internal void myChangeHandler (object o) - { - if (myLastArrivedPacket != null) - mySubsequentChangeHandler ((DataPacket)o); - else - { - myLastArrivedPacket = (DataPacket)o; - myFirstChangeHandler ((DataPacket)o); - } - } +// internal void myChangeHandler (object o) +// { +// if (myLastArrivedPacket != null) +// mySubsequentChangeHandler ((DataPacket)o); +// else +// { +// myLastArrivedPacket = (DataPacket)o; +// myFirstChangeHandler ((DataPacket)o); +// } +// } - private void myFirstChangeHandler(DataPacket dp) - { - myMaster.startSession (); - myMaster.add (dp.newRows, myIsLeft); - myMaster.endSession (); - } +// private void myFirstChangeHandler(DataPacket dp) +// { +// myMaster.startSession (); +// myMaster.add (dp.newRows, myIsLeft); +// myMaster.endSession (); +// } - private void mySubsequentChangeHandler (DataPacket dp) - { - myMaster.startSession (); - //myMaster.delete (dp.deletedRows, myIsLeft); - //myMaster.add (dp.addedRows, myIsLeft); - myMaster.mergeChanges (dp.deletedRows, dp.addedRows, myIsLeft); - myMaster.endSession (); - } +// private void mySubsequentChangeHandler (DataPacket dp) +// { +// myMaster.startSession (); +// //myMaster.delete (dp.deletedRows, myIsLeft); +// //myMaster.add (dp.addedRows, myIsLeft); +// myMaster.mergeChanges (dp.deletedRows, dp.addedRows, myIsLeft); +// myMaster.endSession (); +// } - public IBlueTableSchema opSchema - { - get - { - if (myLastArrivedPacket == null || myLastArrivedPacket.newRows == null) - return null; - return myLastArrivedPacket.newRows.schema; - } - } +// public IBlueTableSchema opSchema +// { +// get +// { +// if (myLastArrivedPacket == null || myLastArrivedPacket.newRows == null) +// return null; +// return myLastArrivedPacket.newRows.schema; +// } +// } - public void init () - { - myLastArrivedPacket = null; - } - } -} +// public void init () +// { +// myLastArrivedPacket = null; +// } +// } +//} diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/UnionStream.cs b/src/DataStreamSuite/DataStream/Primitives/Union/UnionStream.cs index 5af486f8f..0d4cb5f82 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Union/UnionStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Union/UnionStream.cs @@ -1,119 +1,119 @@ -using System; -using System.Data; -using System.Collections; -using BCG.Utilities; -using BCG.Utilities.General; -namespace BCG -{ - /// - /// Provides a live stream of Union of 2 streams - /// - public abstract class UnionStream : ViewStream - { - protected UnionOperand myLeftOp, myRightOp; - protected ViewStream myLeftVs, myRightVs; - protected DataPacket myLastFiredPacket; - protected IDictionary myLeftIndex = new Hashtable (), myRightIndex = new Hashtable (); - protected int[] myLeftIndexToIndexMap, myRightIndexToIndexMap; - protected int myIdCounter, myIdIndex; +//using System; +//using System.Data; +//using System.Collections; +//using BCG.Utilities; +//using BCG.Utilities.General; +//namespace BCG +//{ +// /// +// /// Provides a live stream of Union of 2 streams +// /// +// public abstract class UnionStream : ViewStream +// { +// protected UnionOperand myLeftOp, myRightOp; +// protected ViewStream myLeftVs, myRightVs; +// protected DataPacket myLastFiredPacket; +// protected IDictionary myLeftIndex = new Hashtable (), myRightIndex = new Hashtable (); +// protected int[] myLeftIndexToIndexMap, myRightIndexToIndexMap; +// protected int myIdCounter, myIdIndex; - protected override void myOnFirstSubscriber() - { - myLeftOp = new UnionOperand (true, this); - myRightOp = new UnionOperand (false, this); - myLeftVs.onChange += new DataStream.ChangeDelegate (myLeftOp.myChangeHandler); - myRightVs.onChange += new DataStream.ChangeDelegate (myRightOp.myChangeHandler); - } +// protected override void myOnFirstSubscriber() +// { +// myLeftOp = new UnionOperand (true, this); +// myRightOp = new UnionOperand (false, this); +// myLeftVs.onChange += new DataStream.ChangeDelegate (myLeftOp.myChangeHandler); +// myRightVs.onChange += new DataStream.ChangeDelegate (myRightOp.myChangeHandler); +// } - protected override void myOnLastSubscriber () - { - myInit (); - myLeftVs.onChange -= new DataStream.ChangeDelegate (myLeftOp.myChangeHandler); - myRightVs.onChange -= new DataStream.ChangeDelegate (myRightOp.myChangeHandler); - } +// protected override void myOnLastSubscriber () +// { +// myInit (); +// myLeftVs.onChange -= new DataStream.ChangeDelegate (myLeftOp.myChangeHandler); +// myRightVs.onChange -= new DataStream.ChangeDelegate (myRightOp.myChangeHandler); +// } - public void startSession () - { - if (myLastFiredPacket == null) - { - myLastFiredPacket = new DataPacket (); - if (myLeftOp.opSchema != null)//ie call back initiated by left operand - { - myLeftIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myLeftOp.opSchema); - myLastFiredPacket.newRows = new BlueTable (myLeftOp.opSchema); - myIdIndex = myLeftOp.opSchema["id"].colNo; - } - else - { - myRightIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myRightOp.opSchema); - myLastFiredPacket.newRows = new BlueTable (myRightOp.opSchema); - myIdIndex = myRightOp.opSchema["id"].colNo; - } - } - else if (myRightIndexToIndexMap == null && myRightOp.opSchema != null) - myRightIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myRightOp.opSchema); - else if (myLeftIndexToIndexMap == null && myLeftOp.opSchema != null) - myLeftIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myLeftOp.opSchema); - myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema); - myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema); - } +// public void startSession () +// { +// if (myLastFiredPacket == null) +// { +// myLastFiredPacket = new DataPacket (); +// if (myLeftOp.opSchema != null)//ie call back initiated by left operand +// { +// myLeftIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myLeftOp.opSchema); +// myLastFiredPacket.newRows = new BlueTable (myLeftOp.opSchema); +// myIdIndex = myLeftOp.opSchema["id"].colNo; +// } +// else +// { +// myRightIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myRightOp.opSchema); +// myLastFiredPacket.newRows = new BlueTable (myRightOp.opSchema); +// myIdIndex = myRightOp.opSchema["id"].colNo; +// } +// } +// else if (myRightIndexToIndexMap == null && myRightOp.opSchema != null) +// myRightIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myRightOp.opSchema); +// else if (myLeftIndexToIndexMap == null && myLeftOp.opSchema != null) +// myLeftIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myLeftOp.opSchema); +// myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema); +// myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema); +// } - private int[] myInitIndexToIndex (IBlueTableSchema sch1, IBlueTableSchema sch2) - { - //BlueEventLog.logAssert (sch1.columnNames.Length.Equals (sch2.columnNames.Length)); - //foreach (string colName in sch1.columnNames) - // BlueEventLog.logAssert (sch2[colName] != null); - if(!sch1.columnNames.Length.Equals(sch2.columnNames.Length)) - throw new BlueDataException("Union stream: different schemas"); - foreach(string colName in sch1.columnNames) - { - if(sch2[colName] == null) - throw new BlueDataException("Union stream: can't find " + colName); - } - int[] retVal = new int[sch1.columnNames.Length]; - for (int i = 0; i < retVal.Length; i++) - retVal[i] = sch2[sch1[i].colName].colNo; - return retVal; - } +// private int[] myInitIndexToIndex (IBlueTableSchema sch1, IBlueTableSchema sch2) +// { +// //BlueEventLog.logAssert (sch1.columnNames.Length.Equals (sch2.columnNames.Length)); +// //foreach (string colName in sch1.columnNames) +// // BlueEventLog.logAssert (sch2[colName] != null); +// if(!sch1.columnNames.Length.Equals(sch2.columnNames.Length)) +// throw new BlueDataException("Union stream: different schemas"); +// foreach(string colName in sch1.columnNames) +// { +// if(sch2[colName] == null) +// throw new BlueDataException("Union stream: can't find " + colName); +// } +// int[] retVal = new int[sch1.columnNames.Length]; +// for (int i = 0; i < retVal.Length; i++) +// retVal[i] = sch2[sch1[i].colName].colNo; +// return retVal; +// } - public virtual void endSession () - { - myFireEvent (myLastFiredPacket.Clone ()); - } +// public virtual void endSession () +// { +// myFireEvent (myLastFiredPacket.Clone ()); +// } - public abstract void add (IBlueRowCollection rows, bool isLeft); +// public abstract void add (IBlueRowCollection rows, bool isLeft); - public abstract void delete (IBlueRowCollection rows, bool isLeft); +// public abstract void delete (IBlueRowCollection rows, bool isLeft); - public abstract void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft); +// public abstract void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft); - private void myInit () - { - myLastFiredPacket = null; - myLeftOp.init (); - myRightOp.init (); - myLeftIndex.Clear (); - myRightIndex.Clear (); - myIdCounter = myIdIndex = 0; - myLeftIndexToIndexMap = myRightIndexToIndexMap = null; - } +// private void myInit () +// { +// myLastFiredPacket = null; +// myLeftOp.init (); +// myRightOp.init (); +// myLeftIndex.Clear (); +// myRightIndex.Clear (); +// myIdCounter = myIdIndex = 0; +// myLeftIndexToIndexMap = myRightIndexToIndexMap = null; +// } - /// - /// Used for test purposes only. Users of this class specifically, should ignore it - /// (too bad, the NUnit cant see the private methods) - /// - /// - /// - internal void receiveStream (DataPacket dp, bool isLeft) - { - if (isLeft) - myLeftOp.myChangeHandler (dp); - else - myRightOp.myChangeHandler (dp); - } - public void receiveStreamTest(DataPacket dp, bool isLeft) - { - receiveStream(dp, isLeft); - } - } -} \ No newline at end of file +// /// +// /// Used for test purposes only. Users of this class specifically, should ignore it +// /// (too bad, the NUnit cant see the private methods) +// /// +// /// +// /// +// internal void receiveStream (DataPacket dp, bool isLeft) +// { +// if (isLeft) +// myLeftOp.myChangeHandler (dp); +// else +// myRightOp.myChangeHandler (dp); +// } +// public void receiveStreamTest(DataPacket dp, bool isLeft) +// { +// receiveStream(dp, isLeft); +// } +// } +//} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs index 3ddf531f3..bba9e532c 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs @@ -114,6 +114,15 @@ namespace BCG { rv.Add(r.get(col)); return rv; } + public static IEnumerable get_colvalues(IEnumerable rows, int start_idx, int n_items,string col) { + List rv = new List(); + int end_idx = Math.Min(start_idx + n_items, rows.Count()); + for (int i = start_idx; i < end_idx; i++) { + var r = rows.ElementAt(i); + rv.Add(r.get(col)); + } + return rv; + } public static IEnumerable get_colvalues(IEnumerable rows, string col, int start_index, int count) { List rv = new List(); int i = 0; diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricalBarsStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricalBarsStream.cs index 97535fb80..e56f6a7fe 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricalBarsStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricalBarsStream.cs @@ -147,7 +147,7 @@ namespace bts.BlueOrb { } } - [DataStreamUserRespresentation("FutureHistoricalBars")] + [DataStreamUserRespresentation("FutureHistoricalBarsStream")] public class FutureHistoricalBarsStream : ViewStream { public FutureHistoricalBarsStream(string fut_contract, string range_beg, string range_end) { DateTime dt_range_beg = DateTime.Parse(range_beg); diff --git a/src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs b/src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs index 91bb628d2..8a60b1c9b 100644 --- a/src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs +++ b/src/DataStreamSuite/SharedStreams/Business/ContractUtils.cs @@ -1,8 +1,8 @@ using bts.SilverPine; -namespace bts.SilverPine { - using bts.api; - using bts.spark; +namespace bts.BlueOrb { + //using bts.api; + //using bts.spark; using System; @@ -16,7 +16,7 @@ namespace bts.SilverPine { public string contract_text; } public class FutureSpreadContract { - public List> legs; + public List> legs; public string symbol { get { return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol; @@ -206,7 +206,7 @@ namespace bts.SilverPine { } // tehre are some non calendar spreads we cant parse //Log.assert(false); - return bts.SilverPine.ContractUtils.ContractType.FutureSpread; + return ContractUtils.ContractType.FutureSpread; } public static ContractCategory get_category(string symbol) { diff --git a/src/DataStreamSuite/SharedStreams/Business/EventsPriceActionStream.cs b/src/DataStreamSuite/SharedStreams/Business/EventsPriceActionStream.cs index cf61fb4eb..876921543 100644 --- a/src/DataStreamSuite/SharedStreams/Business/EventsPriceActionStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/EventsPriceActionStream.cs @@ -1,23 +1,23 @@ -namespace bts.SilverPine { - using BCG; +//namespace bts.BlueOrb { +// using BCG; - // Finds the regions around specified events - [DataStreamUserRespresentation("EventsPriceAction")] - public class EventsPriceActionStream : ViewStream { - public EventsPriceActionStream(string symbol, string event_type, string pre_event_period, - string post_event_period, int n_events) : - this(symbol, event_type, TimeBasedFilter.parse_duration(pre_event_period), TimeBasedFilter.parse_duration(post_event_period), - n_events, ViewStreamDatabase.instance) { } +// // Finds the regions around specified events +// [DataStreamUserRespresentation("EventsPriceAction")] +// public class EventsPriceActionStream : ViewStream { +// public EventsPriceActionStream(string symbol, string event_type, string pre_event_period, +// string post_event_period, int n_events) : +// this(symbol, event_type, TimeBasedFilter.parse_duration(pre_event_period), TimeBasedFilter.parse_duration(post_event_period), +// n_events, ViewStreamDatabase.instance) { } - public EventsPriceActionStream(string symbol, string event_type, TimeSpan pre_event_period, - TimeSpan post_event_period, int n_events, ViewStreamDatabase vsd) { - // TODO: figure out which bars to use, 1 min or 1 day or live, based on pre_event_period / post_event_period - FuturesActiveContractStream olhcv_str = new FuturesActiveContractStream(symbol); - ViewStream events_str = new EventsStream(vsd, event_type); - // todo - //events_str = new TopStream(events_str, n_events, "-ts"); - ViewStream rs = new TimeProximityJoinStream(olhcv_str, "ts_event", events_str, "ts", pre_event_period, post_event_period); - mySetRealSource(rs); - } - } -} \ No newline at end of file +// public EventsPriceActionStream(string symbol, string event_type, TimeSpan pre_event_period, +// TimeSpan post_event_period, int n_events, ViewStreamDatabase vsd) { +// // TODO: figure out which bars to use, 1 min or 1 day or live, based on pre_event_period / post_event_period +// FuturesActiveContractStream olhcv_str = new FuturesActiveContractStream(symbol); +// ViewStream events_str = new EventsStream(vsd, event_type); +// // todo +// //events_str = new TopStream(events_str, n_events, "-ts"); +// ViewStream rs = new TimeProximityJoinStream(olhcv_str, "ts_event", events_str, "ts", pre_event_period, post_event_period); +// mySetRealSource(rs); +// } +// } +//} diff --git a/src/DataStreamSuite/SharedStreams/Business/Indicators/IndicatorStream.cs b/src/DataStreamSuite/SharedStreams/Business/Indicators/IndicatorStream.cs index f6aac792f..dfa334a75 100644 --- a/src/DataStreamSuite/SharedStreams/Business/Indicators/IndicatorStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/Indicators/IndicatorStream.cs @@ -1,20 +1,20 @@ -namespace bts.SilverPine { - using BCG; +//namespace bts.SilverPine { +// using BCG; - public class IndicatorStream : ViewStream { - enum Indicator { - StandardDeviation = 0 - } - public class IndicatorInfo { - Indicator indicator; - IDictionary parameters; - } - public IndicatorStream(string symbol_name, string indicator_name, string indicator_params_json) { +// public class IndicatorStream : ViewStream { +// enum Indicator { +// StandardDeviation = 0 +// } +// public class IndicatorInfo { +// Indicator indicator; +// IDictionary parameters; +// } +// public IndicatorStream(string symbol_name, string indicator_name, string indicator_params_json) { - } - public IndicatorStream(string symbol_name, IndicatorInfo info) { +// } +// public IndicatorStream(string symbol_name, IndicatorInfo info) { - } +// } - } -} \ No newline at end of file +// } +//} diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/AbsoluteVolumeStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/AbsoluteVolumeStream.cs index 5bf3e32bf..506c0fe23 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/AbsoluteVolumeStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/AbsoluteVolumeStream.cs @@ -1,57 +1,57 @@ -namespace bts.SilverPine { - using BCG; +//namespace bts.SilverPine { +// using BCG; - using ZstdSharp.Unsafe; +// using ZstdSharp.Unsafe; - [DataStreamUserRespresentation("AbsVolume")] - // Calculates the volume within a time window (this does not do rates, like highest daily volume etc, - // but with some thought can perhaps be extended) - public class AbsoluteVolumeStream : ViewStream { - // TODO, abstract this out - // adds contract_type to OHLCV stream - private static ViewStream _get_ohlcv_steam(string symbol, DateTime d1, DateTime d2) { - var ohlcv_params = OHLCVStream.get_params(d1, d2); - ViewStream rs = new OHLCVStream(symbol, ohlcv_params); - Func create_schema = - s => BlueDataUtility.add_columns(s, Tuple.Create(typeof(string), "contract_type")); - rs = new SuperMapStream(rs, - s => create_schema(s), - r => { - IBlueTableSchema out_schema = create_schema(r.schema); - string contract_txt = r.get("contract"); - var contr_type = ContractUtils.get_type(contract_txt); - MutableBlueRow out_row = new MutableBlueRow(out_schema, r); - out_row.setValue("contract_type", contr_type.ToString()); - return new IBlueRow[] { out_row }; - }); - return rs; - } +// [DataStreamUserRespresentation("AbsVolume")] +// // Calculates the volume within a time window (this does not do rates, like highest daily volume etc, +// // but with some thought can perhaps be extended) +// public class AbsoluteVolumeStream : ViewStream { +// // TODO, abstract this out +// // adds contract_type to OHLCV stream +// private static ViewStream _get_ohlcv_steam(string symbol, DateTime d1, DateTime d2) { +// var ohlcv_params = OHLCVStream.get_params(d1, d2); +// ViewStream rs = new OHLCVStream(symbol, ohlcv_params); +// Func create_schema = +// s => BlueDataUtility.add_columns(s, Tuple.Create(typeof(string), "contract_type")); +// rs = new SuperMapStream(rs, +// s => create_schema(s), +// r => { +// IBlueTableSchema out_schema = create_schema(r.schema); +// string contract_txt = r.get("contract"); +// var contr_type = ContractUtils.get_type(contract_txt); +// MutableBlueRow out_row = new MutableBlueRow(out_schema, r); +// out_row.setValue("contract_type", contr_type.ToString()); +// return new IBlueRow[] { out_row }; +// }); +// return rs; +// } - public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1) - : this(symbol, ctype, d1, d1) { } +// public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1) +// : this(symbol, ctype, d1, d1) { } - public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1, DateTime d2) - : this(symbol, Enum.Parse(ctype), d1, d2) { - } +// public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1, DateTime d2) +// : this(symbol, Enum.Parse(ctype), d1, d2) { +// } - static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) { - if (dt1 < dt2) { - DateTime aux = dt1; - dt1 = dt2; - dt2 = aux; - } - } +// static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) { +// if (dt1 < dt2) { +// DateTime aux = dt1; +// dt1 = dt2; +// dt2 = aux; +// } +// } - private AbsoluteVolumeStream(string symbol, ContractUtils.ContractType ctype, DateTime d1, DateTime d2) { - swap_if_necessary(ref d1, ref d2); - ViewStream ohlcv_str = _get_ohlcv_steam(symbol, d1, d2); - ohlcv_str = new FilterStream(ohlcv_str, - r => r.get_datetime("ts_event") <= d1 && r.get_datetime("ts_event") >= d2); - ohlcv_str = new FilterStream(ohlcv_str, r => String.Compare(ctype.ToString(), r.get("contract_type"), true) == 0); - ViewStream rs = new MaxStream(ohlcv_str, "volume"); - rs = new NewProjectStream(rs, "volume:total_volume contract !"); - mySetRealSource(rs); - } +// private AbsoluteVolumeStream(string symbol, ContractUtils.ContractType ctype, DateTime d1, DateTime d2) { +// swap_if_necessary(ref d1, ref d2); +// ViewStream ohlcv_str = _get_ohlcv_steam(symbol, d1, d2); +// ohlcv_str = new FilterStream(ohlcv_str, +// r => r.get_datetime("ts_event") <= d1 && r.get_datetime("ts_event") >= d2); +// ohlcv_str = new FilterStream(ohlcv_str, r => String.Compare(ctype.ToString(), r.get("contract_type"), true) == 0); +// ViewStream rs = new MaxStream(ohlcv_str, "volume"); +// rs = new NewProjectStream(rs, "volume:total_volume contract !"); +// mySetRealSource(rs); +// } - } -} \ No newline at end of file +// } +//} diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs index e71ff7911..2985d8e81 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs @@ -1,4 +1,4 @@ -namespace bts.SilverPine { +namespace bts.BlueOrb { using BCG; using SharpCompress.Common.Rar; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs index 87f1a3d09..4c59e7cea 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs @@ -1,4 +1,4 @@ -namespace bts.SilverPine { +namespace bts.BlueOrb { using BCG; using bts.api; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs index f43f3a0e7..76fca12e3 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs @@ -1,4 +1,4 @@ -namespace bts.SilverPine { +namespace bts.BlueOrb { using BCG; using System.Diagnostics; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatArbsWeightLinearRegressionStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatArbsWeightLinearRegressionStream.cs index 23f174384..3ad739079 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatArbsWeightLinearRegressionStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatArbsWeightLinearRegressionStream.cs @@ -1,4 +1,4 @@ -namespace bts.SilverPine{ +namespace bts.BlueOrb{ using BCG; public class SpreadStatArbsWeightLinearRegressionStream : ViewStream { private IBlueTableSchema _schema; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs index a229ea38e..20630b236 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs @@ -1,4 +1,4 @@ -namespace bts.SilverPine { +namespace bts.BlueOrb { using BCG; using MathNet.Numerics.LinearAlgebra.Factorization; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadWeightCalculator.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadWeightCalculator.cs index 7704b7074..36ace0731 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadWeightCalculator.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadWeightCalculator.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using MathNet.Numerics.LinearRegression; -namespace bts.SilverPine { +namespace bts.BlueOrb { public static class SpreadWeightCalculator { /// diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/StandardDeviationStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/StandardDeviationStream.cs index 5fff6091f..192d751ca 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/StandardDeviationStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/StandardDeviationStream.cs @@ -1,40 +1,40 @@ -namespace bts.SilverPine { - using BCG; +//namespace bts.SilverPine { +// using BCG; - using MathNet.Numerics.Statistics; +// using MathNet.Numerics.Statistics; - // calculates the reazlied standard deviation of a series of prices - public class StandardDeviationStream : ViewStream { - public StandardDeviationStream(string symbol, string std_dev_period) - : - this(symbol, TimeBasedFilter.parse_duration(std_dev_period)) { } +// // calculates the reazlied standard deviation of a series of prices +// public class StandardDeviationStream : ViewStream { +// public StandardDeviationStream(string symbol, string std_dev_period) +// : +// this(symbol, TimeBasedFilter.parse_duration(std_dev_period)) { } - public StandardDeviationStream(string symbol, TimeSpan std_dev_period) { - // since this is a 1_day bar str, 90 rows => 90 days, so no need to filter - int n_days = std_dev_period.Days; - ViewStream ohlcv_str = new FuturesActiveContractStream(symbol); - IBlueTableSchema out_schema = null; - int id = 0; - Func, uint, IBlueRow> find_std_dev = (rows, idx) => { - int iidx = Convert.ToInt32(idx); - if (rows == null || rows.Count == 0) - return null; - IBlueRow ir = rows.ElementAt(iidx); - out_schema = out_schema ?? ir.schema.add_column(TypeCode.Double, "std_dev"); - MutableBlueRow rv = new MutableBlueRow(out_schema, ir); - rv.setValue("std_dev", Double.NaN); - IEnumerable last_n_close_prices = rows.Skip(iidx).Take(n_days).Select(r => r.get("close")); - if (last_n_close_prices.Count() >= n_days) { - double std_dev = MathNet.Numerics.Statistics.Statistics.StandardDeviation(last_n_close_prices); - rv.setValue("std_dev", std_dev); - } - //rv.setValue("id", id++); - return rv; - }; - ViewStream rs = new GlobalMapStream(ohlcv_str, "ts_event", find_std_dev); - rs = new NullFilterStream(rs, "std_dev", true); - mySetRealSource(rs); - } +// public StandardDeviationStream(string symbol, TimeSpan std_dev_period) { +// // since this is a 1_day bar str, 90 rows => 90 days, so no need to filter +// int n_days = std_dev_period.Days; +// ViewStream ohlcv_str = new FuturesActiveContractStream(symbol); +// IBlueTableSchema out_schema = null; +// int id = 0; +// Func, uint, IBlueRow> find_std_dev = (rows, idx) => { +// int iidx = Convert.ToInt32(idx); +// if (rows == null || rows.Count == 0) +// return null; +// IBlueRow ir = rows.ElementAt(iidx); +// out_schema = out_schema ?? ir.schema.add_column(TypeCode.Double, "std_dev"); +// MutableBlueRow rv = new MutableBlueRow(out_schema, ir); +// rv.setValue("std_dev", Double.NaN); +// IEnumerable last_n_close_prices = rows.Skip(iidx).Take(n_days).Select(r => r.get("close")); +// if (last_n_close_prices.Count() >= n_days) { +// double std_dev = MathNet.Numerics.Statistics.Statistics.StandardDeviation(last_n_close_prices); +// rv.setValue("std_dev", std_dev); +// } +// //rv.setValue("id", id++); +// return rv; +// }; +// ViewStream rs = new GlobalMapStream(ohlcv_str, "ts_event", find_std_dev); +// rs = new NullFilterStream(rs, "std_dev", true); +// mySetRealSource(rs); +// } - } -} \ No newline at end of file +// } +//} diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs index 3130669da..4bf57787e 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/BBOStream.cs @@ -1,4 +1,4 @@ -namespace bts.SilverPine { +namespace bts.BlueOrb { using BCG.Utilities.General; using bts.md; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs index 0ca72d8fa..1556bde33 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs @@ -1,5 +1,6 @@ -namespace BCG { +namespace bts.BlueOrb{ using BCG.Utilities.General; + using BCG; using bts.md; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs index 6816cd940..2ec4abc0d 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs @@ -1,4 +1,4 @@ -namespace bts.SilverPine { +namespace bts.BlueOrb { using BCG; [DataStreamUserRespresentation("Events")] @@ -15,4 +15,4 @@ namespace bts.SilverPine { mySetRealSource(rs); } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs index 2425a2fd7..834344e5f 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs @@ -1,4 +1,4 @@ -namespace bts.SilverPine { +namespace bts.BlueOrb{ using BCG; using bts; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs index f8ef41201..0005e9814 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs @@ -1,4 +1,4 @@ -namespace bts.SilverPine { +namespace bts.BlueOrb { using BCG.Utilities.General; using bts.api; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVPriceDiffStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVPriceDiffStream.cs index 61f9983a5..5edbbbae0 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVPriceDiffStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVPriceDiffStream.cs @@ -1,37 +1,37 @@ -namespace bts.SilverPine { - using BCG; +//namespace bts.SilverPine { +// using BCG; - [DataStreamUserRespresentation("OHLCVPriceDiff")] - public class OHLCVPriceDiffStream : ViewStream { - // TODO, abstract this out - private static ViewStream _get_ohlcv_steam(string symbol_or_contract, DateTime d1, DateTime d2) { - var tup = ContractUtils.get_symbol_category(symbol_or_contract); - var ohlcv_params = OHLCVStream.get_params(d1, d2); - var cont_cat = tup.Item1; - string symbol = tup.Item2; - ViewStream rs = new OHLCVStream(symbol, ohlcv_params); - switch (cont_cat) { - case ContractUtils.ContractCategory.FutureString: - case ContractUtils.ContractCategory.SpreadString: - rs = new FilterStream(rs, $"contract = '{symbol_or_contract}'"); - break; - case ContractUtils.ContractCategory.Symbol: - rs = new FuturesActiveContractStream(rs); - break; - } - return rs; - } +// [DataStreamUserRespresentation("OHLCVPriceDiff")] +// public class OHLCVPriceDiffStream : ViewStream { +// // TODO, abstract this out +// private static ViewStream _get_ohlcv_steam(string symbol_or_contract, DateTime d1, DateTime d2) { +// var tup = ContractUtils.get_symbol_category(symbol_or_contract); +// var ohlcv_params = OHLCVStream.get_params(d1, d2); +// var cont_cat = tup.Item1; +// string symbol = tup.Item2; +// ViewStream rs = new OHLCVStream(symbol, ohlcv_params); +// switch (cont_cat) { +// case ContractUtils.ContractCategory.FutureString: +// case ContractUtils.ContractCategory.SpreadString: +// rs = new FilterStream(rs, $"contract = '{symbol_or_contract}'"); +// break; +// case ContractUtils.ContractCategory.Symbol: +// rs = new FuturesActiveContractStream(rs); +// break; +// } +// return rs; +// } - public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2) - : this(symbol_or_contract, d1, d2, "close") { } +// public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2) +// : this(symbol_or_contract, d1, d2, "close") { } - public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2, string col) { - ViewStream rs = _get_ohlcv_steam(symbol_or_contract, d1, d2); - rs = new ColumnDifferenceStream(rs, r => d1.Equals(r.get_datetime("ts_event")), - r => d2.Equals(r.get_datetime("ts_event")), col, col); - rs = new NewProjectStream(rs, "diff !"); - mySetRealSource(rs); - } +// public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2, string col) { +// ViewStream rs = _get_ohlcv_steam(symbol_or_contract, d1, d2); +// rs = new ColumnDifferenceStream(rs, r => d1.Equals(r.get_datetime("ts_event")), +// r => d2.Equals(r.get_datetime("ts_event")), col, col); +// rs = new NewProjectStream(rs, "diff !"); +// mySetRealSource(rs); +// } - } -} \ No newline at end of file +// } +//} diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs index 30581c0bd..192885118 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs @@ -1,4 +1,4 @@ -namespace bts.SilverPine { +namespace bts.BlueOrb { using bts; using BCG; using MongoDB.Driver; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/SyntheticSpreadStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/SyntheticSpreadStream.cs index 141e3ca5a..932a853fc 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/SyntheticSpreadStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/SyntheticSpreadStream.cs @@ -1,8 +1,7 @@ using System; using BCG; -namespace bts.SilverPine -{ +namespace bts.BlueOrb { /* ---------------------------------------------------------------------- *Expression* diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/BarsSearcherStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/BarsSearcherStream.cs index 5626f7114..0e57ce831 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/BarsSearcherStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/TimSeriesOperations/BarsSearcherStream.cs @@ -1,32 +1,32 @@ -namespace bts.SilverPine { - using BCG; +//namespace bts.SilverPine { +// using BCG; - // this is an internal stream. its job is to take an OHLCV stream - // and filter it for - // 1. symbol (table_name and active contract) and - // 2. time (can be a day of the week, month, and a time of the day period) - public class BarsSearcherStream : ViewStream { - public BarsSearcherStream(string symbol, OHLCVStream.OHLCVBarSize bar_size, - string day_of_week, TimeSpan tod_begin, TimeSpan tod_end) { - ViewStream rs = new OHLCVStream(symbol, bar_size); - rs = new DateTimeExpanstionStream(rs, "ts_event"); - rs = new FilterStream(rs, r => { - DateTime ts = r.get_datetime("ts_event"); - return r.get("ts_event_day") == day_of_week.ToString() && - ts.TimeOfDay >= tod_begin && ts.TimeOfDay <= tod_end; - }); - // look at symbol and either filter ir ot do the active contracts - // assuming its symbol - rs = new NonIncJoinStream(rs, "ts_event_date", "open high low close volume contract ts_event", - new FuturesActiveContractStream(symbol), "ts_event", "ts_event:date contract:active_contract"); - // filter it for the active contract only - rs = new FilterStream(rs, r => String.Compare(r.get("contract"), r.get("active_contract")) == 0); - mySetRealSource(rs); - } - public BarsSearcherStream(string symbol, string bar_size, - string day_of_week, string tod_begin, string tod_end) - : - this(symbol, Enum.Parse(bar_size), day_of_week, - TimeSpan.Parse(tod_begin), TimeSpan.Parse(tod_end)) { } - } -} \ No newline at end of file +// // this is an internal stream. its job is to take an OHLCV stream +// // and filter it for +// // 1. symbol (table_name and active contract) and +// // 2. time (can be a day of the week, month, and a time of the day period) +// public class BarsSearcherStream : ViewStream { +// public BarsSearcherStream(string symbol, OHLCVStream.OHLCVBarSize bar_size, +// string day_of_week, TimeSpan tod_begin, TimeSpan tod_end) { +// ViewStream rs = new OHLCVStream(symbol, bar_size); +// rs = new DateTimeExpanstionStream(rs, "ts_event"); +// rs = new FilterStream(rs, r => { +// DateTime ts = r.get_datetime("ts_event"); +// return r.get("ts_event_day") == day_of_week.ToString() && +// ts.TimeOfDay >= tod_begin && ts.TimeOfDay <= tod_end; +// }); +// // look at symbol and either filter ir ot do the active contracts +// // assuming its symbol +// rs = new NonIncJoinStream(rs, "ts_event_date", "open high low close volume contract ts_event", +// new FuturesActiveContractStream(symbol), "ts_event", "ts_event:date contract:active_contract"); +// // filter it for the active contract only +// rs = new FilterStream(rs, r => String.Compare(r.get("contract"), r.get("active_contract")) == 0); +// mySetRealSource(rs); +// } +// public BarsSearcherStream(string symbol, string bar_size, +// string day_of_week, string tod_begin, string tod_end) +// : +// this(symbol, Enum.Parse(bar_size), day_of_week, +// TimeSpan.Parse(tod_begin), TimeSpan.Parse(tod_end)) { } +// } +//} diff --git a/src/DataStreamSuite/SharedStreams/NL/NLStream.cs b/src/DataStreamSuite/SharedStreams/NL/NLStream.cs index 9a714d91b..d3cdf6714 100644 --- a/src/DataStreamSuite/SharedStreams/NL/NLStream.cs +++ b/src/DataStreamSuite/SharedStreams/NL/NLStream.cs @@ -2,34 +2,38 @@ namespace bts.SilverPine { using BCG; using bts; - using bts.SilverPine; + //using bts.SilverPine; - [DataStreamUserRespresentation("nl")] - public class NLStream : ViewStream { - private readonly string _nl_prompt; - private ViewStream _nl_to_dsl_stream, _dsl_stream; - public NLStream(string nl_prompt) { - _nl_prompt = nl_prompt; - } + namespace bts.BlueOrb { - protected override void myOnFirstSubscriber() { - _nl_to_dsl_stream = new NLToDSLStream(_nl_prompt); - _nl_to_dsl_stream.onChange += _handle_dsl_callback; - } + [DataStreamUserRespresentation("nl")] + public class NLStream : ViewStream { + private readonly string _nl_prompt; + private ViewStream _nl_to_dsl_stream, _dsl_stream; + public NLStream(string nl_prompt) { + _nl_prompt = nl_prompt; + } - private void _handle_dsl_callback(object dp_obj) { - DataPacket dp = dp_obj as DataPacket; - var r = dp.newRows.to_list().ElementAt(0); - string dsl = r.get("response"); - Log.Gen.info($"NLStream._handle_dsl_callback dsl: {dsl}"); - _dsl_stream = new DSLStream(dsl); - _dsl_stream.onChange += _handle_data_cb; - } + protected override void myOnFirstSubscriber() { + _nl_to_dsl_stream = new NLToDSLStream(_nl_prompt); + _nl_to_dsl_stream.onChange += _handle_dsl_callback; + } - private void _handle_data_cb(object dp_obj) { - var dp = dp_obj as DataPacket; - DataPacket dp_out = BlueDataUtility.build_dp_inc(dp.newRows, myLastValue as DataPacket); - myFireEvent(dp_out); + private void _handle_dsl_callback(object dp_obj) { + DataPacket dp = dp_obj as DataPacket; + var r = dp.newRows.to_list().ElementAt(0); + string dsl = r.get("response"); + Log.Gen.info($"NLStream._handle_dsl_callback dsl: {dsl}"); + _dsl_stream = new DSLStream(dsl); + _dsl_stream.onChange += _handle_data_cb; + } + + private void _handle_data_cb(object dp_obj) { + var dp = dp_obj as DataPacket; + DataPacket dp_out = BlueDataUtility.build_dp_inc(dp.newRows, myLastValue as DataPacket); + myFireEvent(dp_out); + } } } -} \ No newline at end of file + +} diff --git a/src/client/spark/Risk/PNL/PNLTradeLogRow.cs b/src/client/spark/Risk/PNL/PNLTradeLogRow.cs index f145fecb4..0d08f8cd0 100644 --- a/src/client/spark/Risk/PNL/PNLTradeLogRow.cs +++ b/src/client/spark/Risk/PNL/PNLTradeLogRow.cs @@ -169,6 +169,12 @@ namespace mbts.analytics { return 0; } } + // if < 0 then rebate, > 0 we pay a fee + public double trade_transaction_fee{ + get{ + + } + } public double pos_gamma { get { return tl_row.security_type == SecurityType.Option ?