Polishing BlueOrb streams

This commit is contained in:
Taha committed 2025-07-13 21:21:10 -04:00
1 parent 593601551d
commit 0858eaac23
53 files changed
+1347 -1554

No files matched your search

@@ -0,0 +1,11 @@
namespace bts.BlueOrb {
using BCG;
[DataStreamUserRespresentation("AddColumnsStream")]
public class AddColumnsStream: ViewStream {
public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
mySetRealSource(rs);
}
}
}
@@ -1,7 +1,7 @@
using System;
namespace BCG {
[DataStreamUserRespresentation("count")]
[DataStreamUserRespresentation("CountStream")]
public class CountStream : ViewStream {
private DataPacket myLastFiredDataPacket;
private IBlueTableSchema mySchema = new BlueTableSchema("count",
@@ -3,36 +3,34 @@ namespace bts.BlueOrb {
using System;
using System.Text.RegularExpressions;
public class RegExpFilter : IStreamFilter {
private Regex[] myRegExps;
private string[] myCols;
public RegExpFilter(string col, string regExp)
: this(new string[] { col }, new string[] { regExp }) { }
public RegExpFilter(string[] cols, string[] regExps) {
myCols = cols;
myRegExps = new Regex[regExps.Length];
for (int i = 0; i < regExps.Length; i++)
myRegExps[i] = new Regex(regExps[i]);
}
#region IStreamFilter Members
public bool filter(IBlueRow dr) {
bool rv = true;
for (int i = 0; i < myCols.Length; i++) {
rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
if (!rv)
return rv;
}
return rv;
}
#endregion
}
//[DataStreamUserReprsentation("FilterStream")]
[DataStreamUserRespresentation("FilterStream")]
public class FilterStream : ViewStream {
public class RegExpFilter : IStreamFilter {
private Regex[] myRegExps;
private string[] myCols;
public RegExpFilter(string col, string regExp)
: this(new string[] { col }, new string[] { regExp }) { }
public RegExpFilter(string[] cols, string[] regExps) {
myCols = cols;
myRegExps = new Regex[regExps.Length];
for (int i = 0; i < regExps.Length; i++)
myRegExps[i] = new Regex(regExps[i]);
}
#region IStreamFilter Members
public bool filter(IBlueRow dr) {
bool rv = true;
for (int i = 0; i < myCols.Length; i++) {
rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
if (!rv)
return rv;
}
return rv;
}
#endregion
}
private BlueTable myLastFiredRows;
private IBlueRowCollection myLastReceivedRows;
private IStreamFilter myFilter;
@@ -1,17 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BCG;
namespace bts.BlueOrb {
public interface INonIncJoinInlineMapper
{
IBlueTableSchema get_output_schema(IBlueTableSchema schema);
IBlueRowCollection map_rows(IBlueRowCollection rows);
event MapperChangeDelegate onMapperChange;
}
}
@@ -1,152 +0,0 @@
using bts;
using bts.collections;
using mbts;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Security.Cryptography;
using BCG;
namespace bts.BlueOrb {
internal class NonIncJoinBookkeeper : IDisposable {
public Action on_new_data;
private readonly NonIncJoinSpec _join_spec;
private readonly int _join_position;
private readonly int _start_index_in_output;
private readonly BlueRowHasher _row_hasher;
private readonly bool _is_left_most;
private readonly bool _is_tag_along;
private int _n_total_fires = 0, _n_expensive_fires = 0;
private IBlueTableSchema _schema = null;
private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
private IBlueRow _solitary_row = null;
private int[] _join_col_indices = null,
_data_col_indices = null;
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
_join_spec = the_join_spec;
_join_position = join_position;
_start_index_in_output = start_index_in_output;
_row_hasher = row_hasher;
_is_left_most = _join_position == 0;
_is_tag_along = _join_spec.join_col_names.Length == 0;
}
private bool _filled_missing_data = false;
private void _fill_missing_data(DataPacket dp) {
if (_filled_missing_data)
return;
Func<string[], bool> null_or_empty = (str_arr) => {
if (str_arr == null || str_arr.Length == 0)
return true;
foreach (string s in str_arr) {
if (!String.IsNullOrEmpty(s)) {
return false;
}
}
return true;
};
Func<string[], string[]> remove_bad_cols = (col_names) => {
HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
List<string> cols_list = new List<string>(col_names);
cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
return cols_list.ToArray();
};
if (null_or_empty(_join_spec.data_col_names)) {
_join_spec.data_col_names = dp.schema.columnNames;
}
_join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
if (null_or_empty(_join_spec.data_col_names)) {
_join_spec.data_col_names = dp.schema.columnNames;
}
if (null_or_empty(_join_spec.projected_col_names)) {
_join_spec.projected_col_names = _join_spec.data_col_names;
}
}
public void init() {
_join_spec.stream.onChange += _on_stream_cb;
}
public void _on_stream_cb(object dp_obj) {
//Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
var timer = Stopwatch.StartNew();
DataPacket dp = (DataPacket)dp_obj;
_fill_missing_data(dp);
if (_schema == null) {
_schema = dp.schema;
_join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
_data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
}
if (_is_left_most) {
_row_list.Clear();
foreach (IBlueRow r in dp.newRows)
_row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
} else {
if (_is_tag_along)
_solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
else {
_row_map.Clear();
foreach (IBlueRow r in dp.newRows)
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
}
}
timer.Stop();
_n_total_fires++;
if (timer.ElapsedMilliseconds >= 50) {
_n_expensive_fires++;
Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
+ $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
}
if (on_new_data != null)
on_new_data();
}
public bool is_ready { get { return _schema != null; } }
public void constribute_cols_to_schema(List<ColumnInfo> cis) {
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
}
public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
public void copy_master_row_to_output_row(int row_index, object[] output_row) {
IBlueRow this_row = _row_list[row_index].second;
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
}
public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
if (_is_tag_along) {
if (_solitary_row != null)
BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
else
copy_dbnull_to_output_row(output_row);
} else {
IBlueRow this_row = null;
if (_row_map.TryGetValue(row_hash, out this_row))
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
else
copy_dbnull_to_output_row(output_row);
}
}
private void copy_dbnull_to_output_row(object[] output_row) {
for (int i = 0; i < _data_col_indices.Length; i++)
output_row[i + _start_index_in_output] = DBNull.Value;
}
#region IDisposable
public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
#endregion
}
}
@@ -1,67 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BCG;
using mbts;
namespace bts.BlueOrb {
// this a collection of spec, theres no joining logic in here
public class NonIncJoinSpec {
public ViewStream stream;
// join col names are the names on which the streams are joined (for equality),
// data col names are the names in the original stream
// projected col names are their new names, ie if its x:l_x, then x is in the
// data_col_names and l_x in projected cols
public string[] join_col_names, data_col_names, projected_col_names;
public NonIncJoinSpec() { }
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
: this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
stream = the_stream;
join_col_names = the_join_col_names;
data_col_names = the_data_col_names;
projected_col_names = the_projected_col_names;
}
public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
foreach (Triplet<ViewStream, string, string> r in join_specs) {
string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
string[] visible_cols = r.third.Split(' ');
List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
foreach (string s in visible_cols) {
string[] toks = s.Split(':');
if (toks.Length == 1) {
data_col_names.Add(toks[0]);
mapped_col_names.Add(toks[0]);
} else if (toks.Length == 2) {
data_col_names.Add(toks[0]);
mapped_col_names.Add(toks[1]);
}
}
rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
}
return rv.ToArray();
}
public override string ToString() {
Func<string[], string> arr_txt = (a) => {
string rv = "";
foreach (string s in a) {
rv += "{s} ";
}
rv = rv.Trim();
return rv;
};
string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
return rv;
}
}
}
@@ -1,165 +0,0 @@
using bts;
using mbts;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using BCG;
namespace bts.BlueOrb {
[DataStreamUserRespresentation("JoinStream")]
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
private readonly NonIncJoinSpec[] _join_specs;
private readonly INonIncJoinInlineMapper _output_mapper;
private readonly BlueRowHasher _row_hasher;
private readonly TimeSpan _throttle_period;
private readonly ViewStreamDatabase _vsd;
private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
private IBlueTableSchema _output_schema = null;
private DateTime _last_fire_time = DateTime.MinValue;
private bool _is_dirty = false;
public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
ViewStream right, string right_join_cols, string right_proj)
: this(new Triplet<ViewStream, string, string>[]
{
new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
}) { }
public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
: this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
params Triplet<ViewStream, string, string>[] join_specs)
: this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) {
}
public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
: this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
_join_specs = join_specs;
_output_mapper = output_mapper;
_row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
_throttle_period = throttles_ms == 0 ? TimeSpan.Zero : new TimeSpan(0, 0, 0, 0, throttles_ms);
_vsd = vsd;
}
protected override void myOnFirstSubscriber() {
int data_col_no = 1;
for (int jp = 0; jp < _join_specs.Length; jp++) {
NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
//Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
data_col_no += _join_specs[jp].data_col_names.Length;
s.on_new_data += _stream_callback;
_book_keeping_data.Add(s);
}
if (_output_mapper != null)
_output_mapper.onMapperChange += _on_mapper_change;
_vsd.addWorker(this);
foreach (var bkd in _book_keeping_data)
bkd.init();
}
protected override void myOnLastSubscriber() {
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
d.Dispose();
if (_output_mapper != null)
_output_mapper.onMapperChange -= _on_mapper_change;
_vsd.removeWorker(this);
}
private void _stream_callback() {
//Log.Gen.info("NonIncJoinStream._stream_callback received callback");
_is_dirty = true;
doWork();
}
private void _on_mapper_change() {
_is_dirty = true;
doWork();
}
private bool _is_ready() {
bool rv = true;
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
rv &= d.is_ready;
return rv;
}
private void _construct_schema() {
if (_output_schema == null) {
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
d.constribute_cols_to_schema(cis);
_output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
}
}
private int _n_total_fires = 0, _n_expensive_fires = 0;
private void _form_and_fire() {
//Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
if (_is_ready()) {
_construct_schema();
var timer = Stopwatch.StartNew();
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
List<object[]> output_rows = new List<object[]>();
for (int i = 0; i < left_row_list.Count; i++) {
object[] or = new object[_output_schema.columnInfo.Length];
or[0] = i;
output_rows.Add(or);
}
for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
for (int i = 0; i < left_row_list.Count; i++) {
if (ji == 0)
bk_data.copy_master_row_to_output_row(i, output_rows[i]);
else
bk_data.copy_matching_row_to_output_row(left_row_list[i].first, output_rows[i]);
}
}
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
timer.Stop();
_n_total_fires++;
//if (timer.ElapsedMilliseconds >= 250) {
_n_expensive_fires++;
//string log_msg = $"Join and map time for {_book_keeping_data.Count} "
// + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
//Log.Gen.info(log_msg);
//}
var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket, true);
myFireEvent(dp_out);
//myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
}
}
private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
if (_output_mapper != null) {
IBlueRowCollection output_rows;
var timer = Stopwatch.StartNew();
output_rows = _output_mapper.map_rows(unmapped_rows);
timer.Stop();
if (timer.ElapsedMilliseconds >= 50) {
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
}
return output_rows;
} else {
return unmapped_rows;
}
}
public void doWork() {
//Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
_is_dirty = false;
_last_fire_time = DateTime.Now;
_form_and_fire();
}
}
}
}
@@ -0,0 +1,187 @@
namespace bts.BlueOrb {
using BCG;
using RowHash = System.Int32;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Collections;
using System.Data.Entity.Core.Mapping;
[DataStreamUserRespresentation("JoinStream")]
public class SimpleJoinStream : ViewStream {
public class JoinBookkeepingData {
// initialized in cons
private readonly List<string> _join_col_names;
private readonly bool _is_left;
private readonly BlueRowHasher _hasher;
// initialized on first udpate
private IBlueTableSchema _schema;
private int[] _join_cols;
private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
_join_col_names = join_cols.Where (c => !String.IsNullOrEmpty(c)).ToList ();
_is_left = is_left;
_hasher = row_hasher;
}
public IBlueTableSchema schema => _schema;
public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
public List<IBlueRow> get_matching_rows (int hash) {
List<IBlueRow> rv = null;
_hash_row_map.TryGetValue(hash, out rv);
return rv;
}
private RowHash _get_hash(IBlueRow r) {
if (_join_cols == null || _join_cols.Length == 0) {
return 0;
}
return _hasher.get_hash(r, _join_cols);
}
public void update_data(DataPacket dp) {
if (!ready) {
_schema = dp.newRows.schema;
_join_cols = _join_col_names.Count == 0 ? new int[] { }: _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
}
_hash_row_map.Clear();
foreach (IBlueRow r in dp.newRows) {
int hash = _get_hash(r);
List<IBlueRow> hash_rows = null;
if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
hash_rows = new List<IBlueRow>();
_hash_row_map.Add(hash, hash_rows);
}
hash_rows.Add(r);
}
}
public bool ready {
get {
return _schema != null;
}
}
}
private ViewStream _ls, _rs;
private JoinBookkeepingData _ldata, _rdata;
private ViewStreamDatabase _vsd;
private IBlueTableSchema _schema = null;
public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
_ls = ls;
_rs = rs;
List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
Log.assert(lj_col_list.Count == rj_col_list.Count);
int n_cols = lj_col_list.Count;
BlueRowHasher hasher = new BlueRowHasher(n_cols);
_ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
_rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
}
protected override void myOnFirstSubscriber() {
_ls.onChange += _ls_onchange;
_rs.onChange += _rs_onchange;
}
private void _ls_onchange(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
_ldata.update_data(dp);
_form_and_fire();
}
private void _rs_onchange(object dp_obj) {
DataPacket dp = dp_obj as DataPacket ;
_rdata.update_data(dp);
_form_and_fire();
}
protected override void myOnLastSubscriber() {
_ls.onChange -= _ls_onchange;
_rs.onChange -= _rs_onchange;
}
private void _build_schema() {
if (_schema != null)
return;
List<ColumnInfo> cis = new List<ColumnInfo>();
cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
}
foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
}
_schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
}
private void _form_and_fire() {
if (!_ldata.ready||!_rdata.ready)
return;
_build_schema();
List<object[]> res_rows = new List<object[]>();
Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
int offset = 1;
JoinBookkeepingData sdata = _ldata;
if (is_left) {
offset = 1;
sdata = _ldata;
} else {
sdata = _rdata;
offset = 1 + _ldata.schema.columnInfo.Length;
}
for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
rval[offset + i] = r[i];
}
};
Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
object[] rv = new object[_schema.columnNames.Length];
for (int i = 0; i < _schema.columnNames.Length; i++) {
rv[i] = DBNull.Value;
}
rv[0] = id;
copy_row(rv, l_row, true);
return rv;
};
// this is where the join happens
foreach (var kv in _ldata.rows) {
RowHash hash = kv.Key;
List<IBlueRow> l_rows = kv.Value;
List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
foreach (IBlueRow lr in l_rows) {
if (r_rows != null) {
foreach (IBlueRow r in r_rows) {
object[] rowvals = build_row_array(res_rows.Count, lr);
copy_row(rowvals, r, false);
res_rows.Add(rowvals);
}
} else {
object[] rowvals = build_row_array(res_rows.Count, lr);
res_rows.Add(rowvals);
}
}
}
BlueTable rv = new BlueTable(_schema);
foreach (object[] rval in res_rows) {
rv.addRow(new BlueRow(_schema, rval));
}
var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
myFireEvent(dp);
}
}
}
@@ -49,14 +49,14 @@ namespace bts.BlueOrb {
}
}
}
public class MSecondStream : TimeUnitStream {
internal class MSecondStream : TimeUnitStream {
public MSecondStream(int intervalInms, int size)
: base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
protected override DateTime myRoundTime(DateTime dt) { return dt; }
}
public class DisplayableMillisecondStream : ViewStream, IStreamMapper {
internal class DisplayableMillisecondStream : ViewStream, IStreamMapper {
private IBlueTableSchema mySchema = new BlueTableSchema("ms",
new ColumnInfo("id", 0, typeof(int)),
new ColumnInfo("time", 1, typeof(string)));
@@ -20,13 +20,14 @@ namespace bts.BlueOrb {
Sum,
EdgeDiff,
EdgeDiffPercentage,
//IsEdgeDiff,
}
public class MultiRowOpHelper {
// input + op
public readonly List<Tuple<string, MultiRowOp>> operations;
public static MultiRowOpHelper parse_ops(string text) {
// text is inp_col: op, inp_col: op, inp_col:op
// ex: "open:First, close:Last, high:Max, low:Min, volume:Sum")
List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
string[] calcs = text.Split(',');
foreach (string c in calcs) {
@@ -71,6 +72,35 @@ namespace bts.BlueOrb {
return or;
}
// aggregates multiple rows into one, used by MultiRowRollingMathStream
public static IBlueRow group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
if (rows.Count() == 0) {
return null;
}
IBlueRow ir = rows.ElementAt(start_idx);
IBlueTableSchema in_schema = ir.schema;
IBlueTableSchema out_schema = in_schema;
// out_col vs inp_cal + type
IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
new Dictionary<string, Tuple<string, MultiRowOp>>();
foreach (var t in parameters.operations) {
string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
out_col_map[out_col_name] = t;
}
MutableBlueRow or = new MutableBlueRow(out_schema, ir);
foreach (var kv in out_col_map) {
string out_col_name = kv.Key;
string in_col_name = kv.Value.Item1;
MultiRowOp op = kv.Value.Item2;
IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, start_idx, n_rows, in_col_name);
double comp_res = compute_op(col_vals, op);
or.setValue(out_col_name, comp_res);
or.setValue("id", ir["id"]);
}
return or;
}
// rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
// calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
@@ -107,9 +137,13 @@ namespace bts.BlueOrb {
case MultiRowOp.Min:
return col_vals.Min<double>();
case MultiRowOp.Mean:
return col_vals.Sum() / col_vals.Count();
return MathNet.Numerics.Statistics.Statistics.Mean(col_vals);
case MultiRowOp.Sum:
return col_vals.Sum();
case MultiRowOp.Median:
return MathNet.Numerics.Statistics.Statistics.Median(col_vals);
//case MultiRowOp.Mode:
// return MathNet.Numerics.Statistics.Statistics.(col_vals);
case MultiRowOp.First:
return col_vals.First();
case MultiRowOp.Last:
@@ -0,0 +1,40 @@
namespace bts.BlueOrb {
using BCG;
using System.Collections.Generic;
using System.Net;
[DataStreamUserRespresentation("MultiRowRollingMathStream")]
public class MultiRowRollingMathStream : ViewStream {
private ViewStream _sorted_parent_str;
private MultiRowOpHelper _op;
private int _rolling_window_sz;
public MultiRowRollingMathStream(ViewStream parent_str, string op_txt, string sort_exp, int rolling_window_sz) {
_sorted_parent_str = new SortStream(parent_str, sort_exp);
_op = MultiRowOpHelper.parse_ops(op_txt);
_rolling_window_sz = rolling_window_sz;
}
protected override void myOnFirstSubscriber() {
_sorted_parent_str.onChange += _parent_str_onchange;
}
protected override void myOnLastSubscriber() {
_sorted_parent_str.onChange -= _parent_str_onchange;
}
private void _parent_str_onchange(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
var rows = dp.newRows.to_list();
BlueTable out_table = null;
for (int i = 0; i < rows.Count; i++) {
var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op);
if (out_table == null)
out_table = new BlueTable(out_r.schema);
out_table.addRow(out_r);
}
var dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket, true);
myFireEvent(dp_out);
}
}
}
@@ -1,8 +1,5 @@
namespace bts.BlueOrb {
using BCG;
using ProtoBuf;
using System;
using System.Collections;
using System.Collections.Generic;
@@ -26,7 +23,7 @@ namespace bts.BlueOrb {
#endregion
#region Row with mapipings build in
#region Row with mappings build in
internal class NewProjectRow : IBlueRow {
#region IBlueRow members
private ProjectData _project_data;
@@ -135,38 +132,30 @@ namespace bts.BlueOrb {
return new NewProjectRow(_project_data, row);
}
// this is where most of the logic is, in terms of translating cols
private static ProjectData _create_project_data(IBlueTableSchema input_schema,
string project_expression) {
project_expression = project_expression.Trim();
string[] project_toks = project_expression.Split(' ');
string[] project_toks = project_expression.Split(' ', ',');
HashSet<string> removed_cols = new HashSet<string>();
Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
// by default every col will show up, so if we have the projection as (a:a1)
// it will convert a to a1,
// modifications (6/30/2025):
// only the columns represented here, either mapped
// or not, will show
// examples: if a schema has cols a b c
// project(parent_str, "a:aa ") -> aa b c
// project(parent_str, "a:aa !c") -> aa b
// project(parent_str, "a:aa !") -> aa
// project(parent_str, "a:aa b")) -> aa b
bool hide_all = false;
foreach (string p_tok in project_toks) {
if (p_tok.StartsWith("!")) {
if (p_tok.Length == 1) {
// if it is a "blanket "!"=> hide all cols except the one that
// are stated explicitly
hide_all = true;
} else {
string removed_col = p_tok.Substring(1);
removed_cols.Add(removed_col);
}
} else if (p_tok.Contains(":")) {
// project(parent_str, "a:aa b") -> aa b
foreach (string o_p_tok in project_toks) {
string p_tok = o_p_tok.Trim();
if (p_tok.Contains(":")) {
// explicit mapping
string[] map_toks = p_tok.Split(":");
string orig_col = map_toks[0];
string mapped_col = map_toks[1];
mapped_to_orig_map.Add(mapped_col, orig_col);
orig_to_mapped.Add(orig_col, mapped_col);
orig_to_mapped.Add(orig_col, mapped_col);
} else {
// this is a col_name explicitely stated
if (input_schema.columnNames.Contains(p_tok)) {
string orig_col = p_tok;
string mapped_col = orig_col;
@@ -181,27 +170,19 @@ namespace bts.BlueOrb {
if (ci.colName == "id") {
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
new_cols.Add(nc);
continue;
}
if (removed_cols.Contains(ci.colName)) {
continue;
continue;
} else if (orig_to_mapped.ContainsKey(ci.colName)) {
string mapped_name = orig_to_mapped[ci.colName];
ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
new_cols.Add(nc);
} else {
if (hide_all)
continue;
//if (ci.colName == "id" || removed_cols.Count > 0) {
// unless columns are explicitely hidden, all columns show up
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
new_cols.Add(nc);
//}
}
}
IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
var mapped_to_orig_map = orig_to_mapped.ToDictionary(kv => kv.Value, kv => kv.Key);
return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
}
}
}
@@ -0,0 +1,92 @@
namespace bts.BlueOrb {
using BCG;
using System.IO;
using System.Diagnostics;
using System;
[DataStreamUserRespresentation("PythonDataframeStream")]
public class PythonDataframeStream : ViewStream {
private ViewStream _parent_str;
private readonly string _parentstream_csv_file_name;
public PythonDataframeStream(ViewStream parent_str) {
string id = System.Guid.NewGuid();
_parentstream_csv_file_name = $"pdfs_in_{id}";
_parent_str = new CSVSaverStream(parent_str, _parentstream_csv_file_name);
//_simple_test();
}
protected override void myOnFirstSubscriber() {
_parent_str.onChange += _parent_str_onchange;
}
protected override void myOnLastSubscriber() {
_parent_str.onChange -= _parent_str_onchange;
}
private void _parent_str_onchange(object) {
throw new NotImplementedException();
}
private void _compute() {
string py_func_wrapper_main = @"
import sys
from user_code import test
if __name__ == '__main__':
inp_file_name = sys.argv[1]
output_file_name = sys.argv[2]
# 1. load inp_file_name into df
# 2. do some test calc
# 3. write the output of 2 into output_file_name
rv = test(arg, out_path)
rv = rv + 12
with open(out_path, 'w') as f:
f.write(str(rv))";
}
private void _simple_test() {
// actual function that needs to be called ////////////////////////
string py_real_function = @"
def test(x, out_path):
rv = x * 10
with open(out_path, 'w') as f:
f.write(str(rv))
return rv";
// the function is in a string object.
File.WriteAllText("user_code.py", py_real_function);
// runner app (top level main function) //////////////////////////////
string py_func_wrapper_main = @"
import sys
from user_code import test
if __name__ == '__main__':
arg = int(sys.argv[1])
out_path = sys.argv[2]
rv = test(arg, out_path)
rv = rv + 12
with open(out_path, 'w') as f:
f.write(str(rv))";
File.WriteAllText("runner.py", py_func_wrapper_main);
string outputPath = "py_out.txt";
ProcessStartInfo psi = new ProcessStartInfo {
FileName = "python",
Arguments = $"runner.py 4 {outputPath}",
RedirectStandardOutput = false,
UseShellExecute = false
};
var proc = Process.Start(psi);
proc.WaitForExit();
// Now read the output written by the function
string result = File.ReadAllText(outputPath);
Console.WriteLine(result); // Should print "6"
}
}
}
@@ -0,0 +1,17 @@
namespace bts.BlueOrb {
using BCG;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
[DataStreamUserRespresentation("SortStream")]
public class SortStream: ViewStream {
public SortStream (ViewStream parent_stream, string sort_expr) {
ViewStream rs = new BCG.SortStream(parent_stream, sort_expr);
mySetRealSource(rs);
}
}
}
@@ -0,0 +1,16 @@
namespace bts.BlueOrb {
using BCG;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
[DataStreamUserRespresentation("TimeBasedRollingFilterStream")]
public class TimeBasedRollingFilterStream : ViewStream {
public TimeBasedRollingFilterStream(ViewStream parent_str, string time_col, double n_seconds) {
ViewStream rs = new BCG.TimeBasedFilterStream(ViewStreamDatabase.instance, parent_str, time_col, TimeSpan.FromSeconds(n_seconds));
mySetRealSource(rs);
}
}
}
@@ -2,7 +2,7 @@ using System;
using BCG;
namespace bts.SilverPine {
namespace bts.BlueOrb {
// Keeps adding changing rows to the stream, hence preserving the history
// and creating a time series, for example this can be used to make bbo updates
// into a time series of prices/qty
@@ -1,261 +1,16 @@
using System;
using System.Collections.Generic;
using mbts;
using BCG;
namespace bts.BlueOrb {
internal class MutableAppendTable : IBlueRowCollection {
private class MyRow : IBlueRow {
private IBlueRow myRow;
private int myID;
using System;
using System.Collections.Generic;
public MyRow(IBlueRow r, int id) {
myRow = r;
myID = id;
}
using mbts;
using BCG;
#region IBlueRow Members
public IBlueTableSchema schema { get { return myRow.schema; } }
public object this[int index] {
get {
if (myRow.schema[index].colName == "id")
return myID;
else
return myRow[index];
}
}
public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
#endregion
#region IDictionary Members
public void Add(object key, object value) { throw new Exception("Row is read only."); }
public void Clear() { throw new Exception("Row is read only."); }
public bool Contains(object key) { return myRow.Contains(key); }
public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
public bool IsFixedSize { get { return true; } }
public bool IsReadOnly { get { return true; } }
public System.Collections.ICollection Keys { get { return myRow.Keys; } }
public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
public System.Collections.ICollection Values {
get {
object[] vals = new object[myRow.Count];
CopyTo(vals, 0);
return vals;
}
}
public object this[object key] {
get { return key is string ? this[key as string] : this[(int)key]; }
set { throw new Exception("The method or operation is not implemented."); }
}
#endregion
#region ICollection Members
public void CopyTo(Array array, int index) {
myRow.CopyTo(array, index);
array.SetValue(myID, index + myRow.schema["id"].colNo);
}
public int Count { get { return myRow.Count; } }
public bool IsSynchronized { get { return myRow.IsSynchronized; } }
public object SyncRoot { get { return myRow.SyncRoot; } }
#endregion
#region IEnumerable Members
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
#endregion
}
private BlueTable myRows;
private int myID = 0;
/// <summary>
/// maps the table no to a dictionary containing the
/// local IDs to a row in the final big table
/// </summary>
private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
public IBlueRow addRow(IBlueRow r, int tableNo) {
if (!myIDMap.ContainsKey(tableNo))
myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
myRows.addRow(nr);
myIDMap[tableNo].Add((int)r["id"], nr);
return nr;
}
public IBlueRow removeRow(IBlueRow r, int tableNo) {
IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
myRows.deleteRow(dr);
myIDMap[tableNo].Remove((int)r["id"]);
return dr;
}
public IBlueRowCollection removeTable(int tableNo) {
if (!myIDMap.ContainsKey(tableNo))
return null;
else {
BlueTable rv = new BlueTable(myRows.schema);
foreach (IBlueRow r in myIDMap[tableNo].Values)
rv.addRow(r);
myRows.deleteRows(rv);
myIDMap.Remove(tableNo);
return rv;
}
}
public void clear() {
myRows.clear();
myIDMap.Clear();
}
#region IBlueRowCollection Members
public IBlueRow this[int id] { get { return myRows[id]; } }
public IBlueTableSchema schema { get { return myRows.schema; } }
public bool containsID(int id) { return myRows.containsID(id); }
#endregion
#region ICollection Members
public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
public int Count { get { return myRows.Count; } }
public bool IsSynchronized { get { return myRows.IsSynchronized; } }
public object SyncRoot { get { return myRows.SyncRoot; } }
#endregion
#region IEnumerable Members
public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
#endregion
}
/// <summary>
/// MutableStream can append multiple streams
/// as well as give the user the flexibility
/// to append (or remove) more streams at
/// runtime
/// </summary>
}
namespace bts.BlueOrb {
[DataStreamUserRespresentation("UnionStream")]
public class MutableAppendStream : ViewStream {
private bool mySubscriptionActive = false;
private MutableAppendTable myTable;
private int myMovingIndex;
private Dictionary<int, Pair<ViewStream, ViewStream>> myStreamIndex = new Dictionary<int, Pair<ViewStream, ViewStream>>();
private bool _allign_col_orders = true;
public MutableAppendStream(params ViewStream[] streams)
: this(new List<ViewStream>(streams)) { }
public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
: this(allign_col_order, new List<ViewStream>(streams)) { }
public MutableAppendStream(ICollection<ViewStream> streams)
: this(true, streams) { }
public MutableAppendStream(bool allign_col_orders, ICollection<ViewStream> streams) {
_allign_col_orders = allign_col_orders;
foreach (ViewStream vs in streams) {
myMovingIndex++;
ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
if (_allign_col_orders)
pj = new ProjectStream(pj, "!");
myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
}
}
public void addStream(ViewStream vs) {
myMovingIndex++;
///the change name is done to identify the callbacks, and the project to
///make all the schemas col orders the same
ViewStream pj = new ChangeTableNameStream(vs, myMovingIndex.ToString());
if (_allign_col_orders)
pj = new ProjectStream(pj, "!");
myStreamIndex.Add(myMovingIndex, new Pair<ViewStream, ViewStream>(vs, pj));
if (mySubscriptionActive)
pj.onChange += new ChangeDelegate(myHandleStreamChanges);
}
public void removeStream(ViewStream vs) {
int index = -1;
foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex) {
if (p.Value.first == vs) {
index = p.Key;
if (mySubscriptionActive) {
p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
if (myTable != null) {
IBlueRowCollection delRows = myTable.removeTable(p.Key);
if (delRows != null)
myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
}
}
break;
}
if (index > 0)
myStreamIndex.Remove(index);
}
}
protected override void myOnFirstSubscriber() {
mySubscriptionActive = true;
foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
p.Value.second.onChange += new ChangeDelegate(myHandleStreamChanges);
}
protected override void myOnLastSubscriber() {
mySubscriptionActive = false;
foreach (KeyValuePair<int, Pair<ViewStream, ViewStream>> p in myStreamIndex)
p.Value.second.onChange -= new ChangeDelegate(myHandleStreamChanges);
if (myTable != null)
myTable.clear();
}
private void myHandleStreamChanges(object newValue) {
DataPacket dp = newValue as DataPacket;
myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
BlueTable adds = new BlueTable(myTable.schema),
dels = new BlueTable(myTable.schema);
int index = Int32.Parse(dp.schema.tableName);
foreach (IBlueRow r in dp.deletedRows) {
IBlueRow dr = myTable.removeRow(r, index);
dels.addRow(dr);
}
foreach (IBlueRow r in dp.addedRows) {
IBlueRow nr = myTable.addRow(r, index);
adds.addRow(nr);
}
myFireEvent(new DataPacket(myTable, adds, dels));
public class UnionStream : ViewStream {
public UnionStream(ViewStream left_stream, ViewStream right_stream) {
ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);
mySetRealSource(rs);
}
}
}
@@ -1,33 +1,51 @@
using System;
using System.Collections.Generic;
namespace BCG {
namespace BCG {
public class AddColumnsStream : ViewStream {
public static Tuple<string, Type, object> parse_exp(string exp) {
public static Tuple<string, Type, object> parse_single_tok(string exp) {
string[] toks = exp.Split(':');
string col_name = toks[0];
string col_value = toks[1];
string col_name = toks[0].Trim();
string col_value = toks[1].Trim();
Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
return Tuple.Create(col_name, tup.Item1, tup.Item2);
}
public static List<Tuple<string, Type, object>> parse_exp(string exp) {
string[] toks = exp.Split(",");
List<Tuple<string, Type, object>> rv = new List<Tuple<string, Type, object>>();
foreach (string s in toks) {
var tup = parse_single_tok(s.Trim());
rv.Add(tup);
}
return rv;
}
private Tuple<string, Type, object> _new_col;
private List<Tuple<string, Type, object>> _new_col_map;
private IBlueTableSchema _out_schema;
public AddColumnsStream(ViewStream vs, string col_def) {
_new_col = parse_exp(col_def);
_new_col_map = parse_exp(col_def);
ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
mySetRealSource(rs);
}
private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
if (_out_schema == null)
_out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
if (_out_schema == null) {
_out_schema = input_schema;
foreach (var m in _new_col_map) {
_out_schema = BlueDataUtility.add_column(_out_schema, m.Item1, m.Item2);
}
}
return _out_schema;
}
private IBlueRow _map(IBlueRow r) {
MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
rv.setValue(_new_col.Item1, _new_col.Item3);
foreach (var m in _new_col_map) {
rv.setValue(m.Item1, m.Item3);
}
return rv;
}
}
@@ -52,14 +52,14 @@ namespace BCG {
}
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
: this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
: this(vsd, rowTimeColumnName, rowTimeToLive, rowTimeToLive) {
}
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filter_period) {
myVsd = vsd;
_time_col = rowTimeColumnName;
_ttl = rowTimeToLive;
myFilterFrequency = filterFrequency;
_filter_period = filter_period;
myNextFilter = DateTime.MinValue;
// add worker
@@ -92,7 +92,7 @@ namespace BCG {
// fire event
onFilterChange();
myNextFilter = now.Add(myFilterFrequency);
myNextFilter = now.Add(_filter_period);
}
}
#endregion
@@ -108,7 +108,7 @@ namespace BCG {
private ViewStreamDatabase myVsd;
private string _time_col;
private TimeSpan _ttl;
private TimeSpan myFilterFrequency;
private TimeSpan _filter_period;
private DateTime myNextFilter;
#endregion
}
@@ -8,7 +8,7 @@ using bts;
using bts.SilverPine;
namespace BCG {
[DataStreamUserRespresentation("AggregateStream")]
//[DataStreamUserRespresentation("AggregateStream")]
public class NonIncGroupByStream : ViewStream {
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
Func<string[]> _groupby_cols_func;
@@ -1,31 +1,31 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
//using System;
//using System.Collections.Generic;
//using System.Linq;
//using System.Text;
namespace BCG
{
public class FullJoinStream : ViewStream
{
private ViewStream myMainStream;
public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
ViewStream v2, string joinCol2, string cmExp2, string empty_col)
{
myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
myMainStream = new AppendStream(myMainStream, filtered);
}
//namespace BCG
//{
// public class FullJoinStream : ViewStream
// {
// private ViewStream myMainStream;
// public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
// ViewStream v2, string joinCol2, string cmExp2, string empty_col)
// {
// myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
// ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
// filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
// myMainStream = new AppendStream(myMainStream, filtered);
// }
protected override void myOnFirstSubscriber()
{
myMainStream.onChange += myHandleMainStreamChange;
}
// protected override void myOnFirstSubscriber()
// {
// myMainStream.onChange += myHandleMainStreamChange;
// }
private void myHandleMainStreamChange(object newData)
{
IBlueRowCollection result = (newData as DataPacket).newRows;
myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
}
}
}
// private void myHandleMainStreamChange(object newData)
// {
// IBlueRowCollection result = (newData as DataPacket).newRows;
// myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
// }
// }
//}
@@ -12,7 +12,7 @@ using System.Collections.Generic;
using System.Data.SQLite;
namespace bts.SilverPine {
namespace bts.BlueOrb{
public static class SQLiteHelper {
public static string location = "C:\\Users\\m_tah\\development\\data\\";
@@ -1,14 +1,12 @@
namespace bts.SilverPine {
namespace bts.BlueOrb {
using System;
using System.Data;
using System.Data.SQLite;
using System.Diagnostics;
using System.Collections;
using BCG;
[DataStreamUserRespresentation("SQLite")]
[DataStreamUserRespresentation("SQLiteStream")]
public class SQLiteStream : ViewStream {
private readonly string _connection_string, _query;
private Stopwatch _stopwatch = new Stopwatch();
@@ -70,4 +68,5 @@ namespace bts.SilverPine {
}
}
}
@@ -1,5 +1,6 @@
using System;
using BCG;
using BCG.Utilities.General;
using bts;
@@ -87,3 +88,17 @@ namespace BCG {
}
}
}
//namespace bts.BlueOrb {
// [DataStreamUserRespresentation("MillisecondStream")]
// public class MillisecondStream : ViewStream {
// public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
// ViewStream rs = new BCG.MillisecondStream(vsd, interval_msecs, size);
// mySetRealSource(rs);
// }
// public MillisecondStream(int interval_msecs, int size)
// : this(interval_msecs, size) {
// }
// }
//}
@@ -1,124 +1,124 @@
namespace bts.SilverPine {
using BCG;
//namespace bts.SilverPine {
// using BCG;
using bts;
using bts.collections;
// using bts;
// //using bts.collections;
using System;
using System.CodeDom;
using System.Collections.Generic;
using System.Linq;
// using System;
// using System.CodeDom;
// using System.Collections.Generic;
// using System.Linq;
// Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
// so:
// f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
[DataStreamUserRespresentation("ColDiff")]
public class ColumnDifferenceStream : ViewStream {
private readonly ViewStream _base_stream;
private readonly FilterStream.OnFilter _f1, _f2;
private readonly string _c1, _c2;
// // Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
// // so:
// // f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
// [DataStreamUserRespresentation("ColDiff")]
// public class ColumnDifferenceStream : ViewStream {
// private readonly ViewStream _base_stream;
// private readonly FilterStream.OnFilter _f1, _f2;
// private readonly string _c1, _c2;
public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
_base_stream = base_stream;
_f1 = f1;
_f2 = f2;
_c1 = col1;
_c2 = col2;
}
// public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
// _base_stream = base_stream;
// _f1 = f1;
// _f2 = f2;
// _c1 = col1;
// _c2 = col2;
// }
public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
FilterStream.OnFilter f1_del = r => {
BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
return rv.filter(r);
};
FilterStream.OnFilter f2_del = r => {
BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
return rv.filter(r);
};
ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
mySetRealSource(rs);
}
// public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
// FilterStream.OnFilter f1_del = r => {
// BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
// return rv.filter(r);
// };
// FilterStream.OnFilter f2_del = r => {
// BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
// return rv.filter(r);
// };
// ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
// mySetRealSource(rs);
// }
// for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
// we should generlize this for any math operation
public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
// use GlobalMapStream
string diff_col_perc_name = $"{diff_col_name}_percentage";
System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
int minuend_idx = Convert.ToInt32(idx);
var inp_schema = rows[0].schema;
var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
// minuend - subtrahend
IBlueRow minuend_row = rows.ElementAt(minuend_idx);
MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
int subtrahend_index = minuend_idx + row_offset;
double diff_val = Double.NaN, diff_val_perc = Double.NaN;
if (subtrahend_index >= 0) {
try {
IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
} catch (Exception ex) {
Log.assert(false, ex.Message);
}
}
rv.setValue(diff_col_name, diff_val);
rv.setValue(diff_col_perc_name, diff_val_perc);
return rv;
};
ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
if (optional_perc_diff_filter != 0) {
rs = new FilterStream(rs, r => {
double diff_val_perc = r.get<double>(diff_col_perc_name);
double abs_diff_val_perc = Math.Abs(diff_val_perc);
return abs_diff_val_perc >= optional_perc_diff_filter;
});
}
mySetRealSource(rs);
}
// // for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
// // we should generlize this for any math operation
// public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
// // use GlobalMapStream
// string diff_col_perc_name = $"{diff_col_name}_percentage";
// System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
// int minuend_idx = Convert.ToInt32(idx);
// var inp_schema = rows[0].schema;
// var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
// out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
// // minuend - subtrahend
// IBlueRow minuend_row = rows.ElementAt(minuend_idx);
// MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
// int subtrahend_index = minuend_idx + row_offset;
// double diff_val = Double.NaN, diff_val_perc = Double.NaN;
// if (subtrahend_index >= 0) {
// try {
// IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
// diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
// diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
// } catch (Exception ex) {
// Log.assert(false, ex.Message);
// }
// }
// rv.setValue(diff_col_name, diff_val);
// rv.setValue(diff_col_perc_name, diff_val_perc);
// return rv;
// };
// ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
// if (optional_perc_diff_filter != 0) {
// rs = new FilterStream(rs, r => {
// double diff_val_perc = r.get<double>(diff_col_perc_name);
// double abs_diff_val_perc = Math.Abs(diff_val_perc);
// return abs_diff_val_perc >= optional_perc_diff_filter;
// });
// }
// mySetRealSource(rs);
// }
protected override void myOnFirstSubscriber() {
_base_stream.onChange += _handle_base_stream_update;
}
// protected override void myOnFirstSubscriber() {
// _base_stream.onChange += _handle_base_stream_update;
// }
protected override void myOnLastSubscriber() {
_base_stream.onChange -= _handle_base_stream_update;
}
// protected override void myOnLastSubscriber() {
// _base_stream.onChange -= _handle_base_stream_update;
// }
private void _handle_base_stream_update(object dp_obj) {
// could have been done with filter and cartesian prod, and then math
DataPacket dp = dp_obj as DataPacket;
List<IBlueRow> rows = dp.newRows.to_list();
var f1_rows = rows.Where(r => _f1(r));
//Log.assert(f1_rows.Count() == 1);
var f2_rows = rows.Where(r => _f2(r));
//Log.assert(f2_rows.Count() == 1);
if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
IBlueRow f1_r = f1_rows.ElementAt(0);
IBlueRow f2_r = f2_rows.ElementAt(0);
double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
BlueTable out_table = new BlueTable(out_schema);
MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
out_row.setValue("diff", diff);
out_table.addRow(out_row);
DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
myFireEvent(dp_out);
} else {
IBlueRow f1_r = rows.ElementAt(0);
IBlueRow f2_r = rows.ElementAt(0);
double diff = Double.NaN;
IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
BlueTable out_table = new BlueTable(out_schema);
MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
out_row.setValue("diff", diff);
out_table.addRow(out_row);
DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
myFireEvent(dp_out);
}
// private void _handle_base_stream_update(object dp_obj) {
// // could have been done with filter and cartesian prod, and then math
// DataPacket dp = dp_obj as DataPacket;
// List<IBlueRow> rows = dp.newRows.to_list();
// var f1_rows = rows.Where(r => _f1(r));
// //Log.assert(f1_rows.Count() == 1);
// var f2_rows = rows.Where(r => _f2(r));
// //Log.assert(f2_rows.Count() == 1);
// if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
// IBlueRow f1_r = f1_rows.ElementAt(0);
// IBlueRow f2_r = f2_rows.ElementAt(0);
// double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
// IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
// BlueTable out_table = new BlueTable(out_schema);
// MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
// out_row.setValue("diff", diff);
// out_table.addRow(out_row);
// DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
// myFireEvent(dp_out);
// } else {
// IBlueRow f1_r = rows.ElementAt(0);
// IBlueRow f2_r = rows.ElementAt(0);
// double diff = Double.NaN;
// IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
// BlueTable out_table = new BlueTable(out_schema);
// MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
// out_row.setValue("diff", diff);
// out_table.addRow(out_row);
// DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
// myFireEvent(dp_out);
// }
}
}
// }
// }
}
//}
@@ -1,118 +1,118 @@
using System;
using System.Data;
using System.Collections;
using BCG.Utilities;
using BCG.Utilities.General;
namespace BCG
{
/// <summary>
/// AppendStream appends 2 tables with each other, with a generated id having no bearing to the original id.
/// The column ordering the 2 tables need not be the same
/// </summary>
public class AppendStream : UnionStream
{
private bool myUseNewAppend = DataStreamLoadFileConfig.instance.useNewAppend;
public AppendStream(ViewStream v1, ViewStream v2)
{
if (myUseNewAppend)
mySetRealSource(new MutableAppendStream(v1, v2));
else
{
myLeftVs = v1;
myRightVs = v2;
}
}
//using System;
//using System.Data;
//using System.Collections;
//using BCG.Utilities;
//using BCG.Utilities.General;
//namespace BCG
//{
// /// <summary>
// /// AppendStream appends 2 tables with each other, with a generated id having no bearing to the original id.
// /// The column ordering the 2 tables need not be the same
// /// </summary>
// public class AppendStream : UnionStream
// {
// private bool myUseNewAppend = DataStreamLoadFileConfig.instance.useNewAppend;
// public AppendStream(ViewStream v1, ViewStream v2)
// {
// if (myUseNewAppend)
// mySetRealSource(new MutableAppendStream(v1, v2));
// else
// {
// myLeftVs = v1;
// myRightVs = v2;
// }
// }
public AppendStream (params ViewStream[] vss)
{
// public AppendStream (params ViewStream[] vss)
// {
if (myUseNewAppend)
mySetRealSource(new MutableAppendStream(vss));
else
{
if (vss.Length == 1)
mySetRealSource(vss[0]);
else if (vss.Length > 1)
{
AppendStream rv = null;
foreach (ViewStream s in vss)
rv = rv == null ? new AppendStream(new ViewStream[] { s }) : new AppendStream(rv, s);
mySetRealSource(rv);
}
else
//don't do anything if they didnt' supply any viewstreams
mySetRealSource(null);
}
// if (myUseNewAppend)
// mySetRealSource(new MutableAppendStream(vss));
// else
// {
// if (vss.Length == 1)
// mySetRealSource(vss[0]);
// else if (vss.Length > 1)
// {
// AppendStream rv = null;
// foreach (ViewStream s in vss)
// rv = rv == null ? new AppendStream(new ViewStream[] { s }) : new AppendStream(rv, s);
// mySetRealSource(rv);
// }
// else
// //don't do anything if they didnt' supply any viewstreams
// mySetRealSource(null);
// }
}
// }
public override void add (IBlueRowCollection rows, bool isLeft)
{
IDictionary index = isLeft ? myLeftIndex : myRightIndex;
int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
foreach (IBlueRow row in rows)
{
UnionRow newRow = new UnionRow (myLastFiredPacket.newRows.schema, row,
myIdCounter ++, myIdIndex, indexToIndexMap);
((BlueTable)myLastFiredPacket.newRows).addRow (newRow);
index[row["id"]] = newRow;
((BlueTable)(myLastFiredPacket.addedRows)).addRow (newRow);
}
}
// public override void add (IBlueRowCollection rows, bool isLeft)
// {
// IDictionary index = isLeft ? myLeftIndex : myRightIndex;
// int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
// foreach (IBlueRow row in rows)
// {
// UnionRow newRow = new UnionRow (myLastFiredPacket.newRows.schema, row,
// myIdCounter ++, myIdIndex, indexToIndexMap);
// ((BlueTable)myLastFiredPacket.newRows).addRow (newRow);
// index[row["id"]] = newRow;
// ((BlueTable)(myLastFiredPacket.addedRows)).addRow (newRow);
// }
// }
public override void delete (IBlueRowCollection rows, bool isLeft)
{
IDictionary index = isLeft ? myLeftIndex : myRightIndex;
foreach (IBlueRow axedRow in rows)
{
int rowId = (int)axedRow["id"];
IBlueRow row = (IBlueRow)index[rowId];
index.Remove (rowId);
((BlueTable)myLastFiredPacket.newRows).deleteRow (row);
((BlueTable)(myLastFiredPacket.deletedRows)).addRow (row);
}
}
// public override void delete (IBlueRowCollection rows, bool isLeft)
// {
// IDictionary index = isLeft ? myLeftIndex : myRightIndex;
// foreach (IBlueRow axedRow in rows)
// {
// int rowId = (int)axedRow["id"];
// IBlueRow row = (IBlueRow)index[rowId];
// index.Remove (rowId);
// ((BlueTable)myLastFiredPacket.newRows).deleteRow (row);
// ((BlueTable)(myLastFiredPacket.deletedRows)).addRow (row);
// }
// }
public override void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft)
{
IDictionary temp = new Hashtable ();
IDictionary index = isLeft ? myLeftIndex : myRightIndex;
int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
IBlueRow corrRow;
foreach (IBlueRow dRow in dels)
{
corrRow = index[dRow["id"]] as IBlueRow;
(myLastFiredPacket.newRows as BlueTable).deleteRow (corrRow);
(myLastFiredPacket.deletedRows as BlueTable).addRow (corrRow);
index.Remove (dRow["id"]);
if (adds.containsID ((int)dRow["id"]))
{
IBlueRow r = adds[(int)dRow["id"]];
corrRow = new UnionRow (myLastFiredPacket.schema, r, (int)corrRow["id"], myIdIndex, indexToIndexMap);
(myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
(myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
index.Add (r["id"], corrRow);
temp.Add (r["id"], true);
}
}
foreach (IBlueRow aRow in adds)
{
if (!temp.Contains (aRow["id"]))
{
corrRow = new UnionRow (myLastFiredPacket.schema, aRow, myIdCounter++, myIdIndex, indexToIndexMap);
(myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
(myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
index.Add (aRow["id"], corrRow);
}
}
}
}
// public override void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft)
// {
// IDictionary temp = new Hashtable ();
// IDictionary index = isLeft ? myLeftIndex : myRightIndex;
// int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
// IBlueRow corrRow;
// foreach (IBlueRow dRow in dels)
// {
// corrRow = index[dRow["id"]] as IBlueRow;
// (myLastFiredPacket.newRows as BlueTable).deleteRow (corrRow);
// (myLastFiredPacket.deletedRows as BlueTable).addRow (corrRow);
// index.Remove (dRow["id"]);
// if (adds.containsID ((int)dRow["id"]))
// {
// IBlueRow r = adds[(int)dRow["id"]];
// corrRow = new UnionRow (myLastFiredPacket.schema, r, (int)corrRow["id"], myIdIndex, indexToIndexMap);
// (myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
// (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
// index.Add (r["id"], corrRow);
// temp.Add (r["id"], true);
// }
// }
// foreach (IBlueRow aRow in adds)
// {
// if (!temp.Contains (aRow["id"]))
// {
// corrRow = new UnionRow (myLastFiredPacket.schema, aRow, myIdCounter++, myIdIndex, indexToIndexMap);
// (myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
// (myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
// index.Add (aRow["id"], corrRow);
// }
// }
// }
// }
public class MergeStream : ViewStream
{
public MergeStream (ViewStream v1, ViewStream v2, string[] mergeCols)
{
mySetRealSource (new DistinctRowStream (new AppendStream (v1, v2), mergeCols));
}
}
}
// public class MergeStream : ViewStream
// {
// public MergeStream (ViewStream v1, ViewStream v2, string[] mergeCols)
// {
// mySetRealSource (new DistinctRowStream (new AppendStream (v1, v2), mergeCols));
// }
// }
//}
@@ -2,170 +2,171 @@ using System;
using System.Collections.Generic;
using mbts;
//using BCG;
namespace BCG {
internal class MutableAppendTable : IBlueRowCollection {
private class MyRow : IBlueRow {
private IBlueRow myRow;
private int myID;
public MyRow(IBlueRow r, int id) {
myRow = r;
myID = id;
}
#region IBlueRow Members
public IBlueTableSchema schema { get { return myRow.schema; } }
public object this[int index] {
get {
if (myRow.schema[index].colName == "id")
return myID;
else
return myRow[index];
}
}
public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
#endregion
#region IDictionary Members
public void Add(object key, object value) { throw new Exception("Row is read only."); }
public void Clear() { throw new Exception("Row is read only."); }
public bool Contains(object key) { return myRow.Contains(key); }
public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
public bool IsFixedSize { get { return true; } }
public bool IsReadOnly { get { return true; } }
public System.Collections.ICollection Keys { get { return myRow.Keys; } }
public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
public System.Collections.ICollection Values {
get {
object[] vals = new object[myRow.Count];
CopyTo(vals, 0);
return vals;
}
}
public object this[object key] {
get { return key is string ? this[key as string] : this[(int)key]; }
set { throw new Exception("The method or operation is not implemented."); }
}
#endregion
#region ICollection Members
public void CopyTo(Array array, int index) {
myRow.CopyTo(array, index);
array.SetValue(myID, index + myRow.schema["id"].colNo);
}
public int Count { get { return myRow.Count; } }
public bool IsSynchronized { get { return myRow.IsSynchronized; } }
public object SyncRoot { get { return myRow.SyncRoot; } }
#endregion
#region IEnumerable Members
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
#endregion
}
private BlueTable myRows;
private int myID = 0;
/// <summary>
/// maps the table no to a dictionary containing the
/// local IDs to a row in the final big table
/// </summary>
private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
public IBlueRow addRow(IBlueRow r, int tableNo) {
if (!myIDMap.ContainsKey(tableNo))
myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
myRows.addRow(nr);
myIDMap[tableNo].Add((int)r["id"], nr);
return nr;
}
public IBlueRow removeRow(IBlueRow r, int tableNo) {
IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
myRows.deleteRow(dr);
myIDMap[tableNo].Remove((int)r["id"]);
return dr;
}
public IBlueRowCollection removeTable(int tableNo) {
if (!myIDMap.ContainsKey(tableNo))
return null;
else {
BlueTable rv = new BlueTable(myRows.schema);
foreach (IBlueRow r in myIDMap[tableNo].Values)
rv.addRow(r);
myRows.deleteRows(rv);
myIDMap.Remove(tableNo);
return rv;
}
}
public void clear() {
myRows.clear();
myIDMap.Clear();
}
#region IBlueRowCollection Members
public IBlueRow this[int id] { get { return myRows[id]; } }
public IBlueTableSchema schema { get { return myRows.schema; } }
public bool containsID(int id) { return myRows.containsID(id); }
#endregion
#region ICollection Members
public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
public int Count { get { return myRows.Count; } }
public bool IsSynchronized { get { return myRows.IsSynchronized; } }
public object SyncRoot { get { return myRows.SyncRoot; } }
#endregion
#region IEnumerable Members
public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
#endregion
}
/// <summary>
/// <summary>
/// MutableStream can append multiple streams
/// as well as give the user the flexibility
/// to append (or remove) more streams at
/// runtime
/// </summary>
[DataStreamUserRespresentation("UnionStream")]
//[DataStreamUserRespresentation("UnionStream")]
public class MutableAppendStream : ViewStream {
internal class MutableAppendTable : IBlueRowCollection {
private class MyRow : IBlueRow {
private IBlueRow myRow;
private int myID;
public MyRow(IBlueRow r, int id) {
myRow = r;
myID = id;
}
#region IBlueRow Members
public IBlueTableSchema schema { get { return myRow.schema; } }
public object this[int index] {
get {
if (myRow.schema[index].colName == "id")
return myID;
else
return myRow[index];
}
}
public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
#endregion
#region IDictionary Members
public void Add(object key, object value) { throw new Exception("Row is read only."); }
public void Clear() { throw new Exception("Row is read only."); }
public bool Contains(object key) { return myRow.Contains(key); }
public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
public bool IsFixedSize { get { return true; } }
public bool IsReadOnly { get { return true; } }
public System.Collections.ICollection Keys { get { return myRow.Keys; } }
public void Remove(object key) { throw new Exception("The method or operation is not implemented."); }
public System.Collections.ICollection Values {
get {
object[] vals = new object[myRow.Count];
CopyTo(vals, 0);
return vals;
}
}
public object this[object key] {
get { return key is string ? this[key as string] : this[(int)key]; }
set { throw new Exception("The method or operation is not implemented."); }
}
#endregion
#region ICollection Members
public void CopyTo(Array array, int index) {
myRow.CopyTo(array, index);
array.SetValue(myID, index + myRow.schema["id"].colNo);
}
public int Count { get { return myRow.Count; } }
public bool IsSynchronized { get { return myRow.IsSynchronized; } }
public object SyncRoot { get { return myRow.SyncRoot; } }
#endregion
#region IEnumerable Members
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
#endregion
}
private BlueTable myRows;
private int myID = 0;
/// <summary>
/// maps the table no to a dictionary containing the
/// local IDs to a row in the final big table
/// </summary>
private Dictionary<int, Dictionary<int, IBlueRow>> myIDMap = new Dictionary<int, Dictionary<int, IBlueRow>>();
public MutableAppendTable(IBlueTableSchema schema) { myRows = new BlueTable(schema); }
public IBlueRow addRow(IBlueRow r, int tableNo) {
if (!myIDMap.ContainsKey(tableNo))
myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
myRows.addRow(nr);
myIDMap[tableNo].Add((int)r["id"], nr);
return nr;
}
public IBlueRow removeRow(IBlueRow r, int tableNo) {
IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
myRows.deleteRow(dr);
myIDMap[tableNo].Remove((int)r["id"]);
return dr;
}
public IBlueRowCollection removeTable(int tableNo) {
if (!myIDMap.ContainsKey(tableNo))
return null;
else {
BlueTable rv = new BlueTable(myRows.schema);
foreach (IBlueRow r in myIDMap[tableNo].Values)
rv.addRow(r);
myRows.deleteRows(rv);
myIDMap.Remove(tableNo);
return rv;
}
}
public void clear() {
myRows.clear();
myIDMap.Clear();
}
#region IBlueRowCollection Members
public IBlueRow this[int id] { get { return myRows[id]; } }
public IBlueTableSchema schema { get { return myRows.schema; } }
public bool containsID(int id) { return myRows.containsID(id); }
#endregion
#region ICollection Members
public void CopyTo(Array array, int index) { myRows.CopyTo(array, index); }
public int Count { get { return myRows.Count; } }
public bool IsSynchronized { get { return myRows.IsSynchronized; } }
public object SyncRoot { get { return myRows.SyncRoot; } }
#endregion
#region IEnumerable Members
public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); }
#endregion
}
private bool mySubscriptionActive = false;
private MutableAppendTable myTable;
private int myMovingIndex;
@@ -173,7 +174,8 @@ namespace BCG {
private bool _allign_col_orders = true;
public MutableAppendStream(params ViewStream[] streams)
: this(new List<ViewStream>(streams)) { }
: this(new List<ViewStream>(streams)) {
}
public MutableAppendStream(bool allign_col_order, params ViewStream[] streams)
: this(allign_col_order, new List<ViewStream>(streams)) { }
@@ -256,3 +258,13 @@ namespace BCG {
}
}
}
//namespace bts.BlueOrb {
// public class UnionStream : ViewStream {
// public UnionStream(ViewStream left_stream, ViewStream right_stream) {
// ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);
// mySetRealSource(rs);
// }
// }
//}
@@ -1,63 +1,63 @@
using System;
//using System;
namespace BCG
{
/// <summary>
/// Respresents a Union Operand. Hooks up to the Stream delegate, and alerts the UnionStream
/// on receiving updates
/// </summary>
public class UnionOperand
{
private bool myIsLeft;
private DataPacket myLastArrivedPacket;
private UnionStream myMaster;
//namespace BCG
//{
// /// <summary>
// /// Respresents a Union Operand. Hooks up to the Stream delegate, and alerts the UnionStream
// /// on receiving updates
// /// </summary>
// public class UnionOperand
// {
// private bool myIsLeft;
// private DataPacket myLastArrivedPacket;
// private UnionStream myMaster;
public UnionOperand (bool isLeft, UnionStream master)
{
myIsLeft = isLeft;
myMaster = master;
}
// public UnionOperand (bool isLeft, UnionStream master)
// {
// myIsLeft = isLeft;
// myMaster = master;
// }
internal void myChangeHandler (object o)
{
if (myLastArrivedPacket != null)
mySubsequentChangeHandler ((DataPacket)o);
else
{
myLastArrivedPacket = (DataPacket)o;
myFirstChangeHandler ((DataPacket)o);
}
}
// internal void myChangeHandler (object o)
// {
// if (myLastArrivedPacket != null)
// mySubsequentChangeHandler ((DataPacket)o);
// else
// {
// myLastArrivedPacket = (DataPacket)o;
// myFirstChangeHandler ((DataPacket)o);
// }
// }
private void myFirstChangeHandler(DataPacket dp)
{
myMaster.startSession ();
myMaster.add (dp.newRows, myIsLeft);
myMaster.endSession ();
}
// private void myFirstChangeHandler(DataPacket dp)
// {
// myMaster.startSession ();
// myMaster.add (dp.newRows, myIsLeft);
// myMaster.endSession ();
// }
private void mySubsequentChangeHandler (DataPacket dp)
{
myMaster.startSession ();
//myMaster.delete (dp.deletedRows, myIsLeft);
//myMaster.add (dp.addedRows, myIsLeft);
myMaster.mergeChanges (dp.deletedRows, dp.addedRows, myIsLeft);
myMaster.endSession ();
}
// private void mySubsequentChangeHandler (DataPacket dp)
// {
// myMaster.startSession ();
// //myMaster.delete (dp.deletedRows, myIsLeft);
// //myMaster.add (dp.addedRows, myIsLeft);
// myMaster.mergeChanges (dp.deletedRows, dp.addedRows, myIsLeft);
// myMaster.endSession ();
// }
public IBlueTableSchema opSchema
{
get
{
if (myLastArrivedPacket == null || myLastArrivedPacket.newRows == null)
return null;
return myLastArrivedPacket.newRows.schema;
}
}
// public IBlueTableSchema opSchema
// {
// get
// {
// if (myLastArrivedPacket == null || myLastArrivedPacket.newRows == null)
// return null;
// return myLastArrivedPacket.newRows.schema;
// }
// }
public void init ()
{
myLastArrivedPacket = null;
}
}
}
// public void init ()
// {
// myLastArrivedPacket = null;
// }
// }
//}
@@ -1,119 +1,119 @@
using System;
using System.Data;
using System.Collections;
using BCG.Utilities;
using BCG.Utilities.General;
namespace BCG
{
/// <summary>
/// Provides a live stream of Union of 2 streams
/// </summary>
public abstract class UnionStream : ViewStream
{
protected UnionOperand myLeftOp, myRightOp;
protected ViewStream myLeftVs, myRightVs;
protected DataPacket myLastFiredPacket;
protected IDictionary myLeftIndex = new Hashtable (), myRightIndex = new Hashtable ();
protected int[] myLeftIndexToIndexMap, myRightIndexToIndexMap;
protected int myIdCounter, myIdIndex;
//using System;
//using System.Data;
//using System.Collections;
//using BCG.Utilities;
//using BCG.Utilities.General;
//namespace BCG
//{
// /// <summary>
// /// Provides a live stream of Union of 2 streams
// /// </summary>
// public abstract class UnionStream : ViewStream
// {
// protected UnionOperand myLeftOp, myRightOp;
// protected ViewStream myLeftVs, myRightVs;
// protected DataPacket myLastFiredPacket;
// protected IDictionary myLeftIndex = new Hashtable (), myRightIndex = new Hashtable ();
// protected int[] myLeftIndexToIndexMap, myRightIndexToIndexMap;
// protected int myIdCounter, myIdIndex;
protected override void myOnFirstSubscriber()
{
myLeftOp = new UnionOperand (true, this);
myRightOp = new UnionOperand (false, this);
myLeftVs.onChange += new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
myRightVs.onChange += new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
}
// protected override void myOnFirstSubscriber()
// {
// myLeftOp = new UnionOperand (true, this);
// myRightOp = new UnionOperand (false, this);
// myLeftVs.onChange += new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
// myRightVs.onChange += new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
// }
protected override void myOnLastSubscriber ()
{
myInit ();
myLeftVs.onChange -= new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
myRightVs.onChange -= new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
}
// protected override void myOnLastSubscriber ()
// {
// myInit ();
// myLeftVs.onChange -= new DataStream.ChangeDelegate (myLeftOp.myChangeHandler);
// myRightVs.onChange -= new DataStream.ChangeDelegate (myRightOp.myChangeHandler);
// }
public void startSession ()
{
if (myLastFiredPacket == null)
{
myLastFiredPacket = new DataPacket ();
if (myLeftOp.opSchema != null)//ie call back initiated by left operand
{
myLeftIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myLeftOp.opSchema);
myLastFiredPacket.newRows = new BlueTable (myLeftOp.opSchema);
myIdIndex = myLeftOp.opSchema["id"].colNo;
}
else
{
myRightIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myRightOp.opSchema);
myLastFiredPacket.newRows = new BlueTable (myRightOp.opSchema);
myIdIndex = myRightOp.opSchema["id"].colNo;
}
}
else if (myRightIndexToIndexMap == null && myRightOp.opSchema != null)
myRightIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myRightOp.opSchema);
else if (myLeftIndexToIndexMap == null && myLeftOp.opSchema != null)
myLeftIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myLeftOp.opSchema);
myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
}
// public void startSession ()
// {
// if (myLastFiredPacket == null)
// {
// myLastFiredPacket = new DataPacket ();
// if (myLeftOp.opSchema != null)//ie call back initiated by left operand
// {
// myLeftIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myLeftOp.opSchema);
// myLastFiredPacket.newRows = new BlueTable (myLeftOp.opSchema);
// myIdIndex = myLeftOp.opSchema["id"].colNo;
// }
// else
// {
// myRightIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myRightOp.opSchema);
// myLastFiredPacket.newRows = new BlueTable (myRightOp.opSchema);
// myIdIndex = myRightOp.opSchema["id"].colNo;
// }
// }
// else if (myRightIndexToIndexMap == null && myRightOp.opSchema != null)
// myRightIndexToIndexMap = myInitIndexToIndex (myLeftOp.opSchema, myRightOp.opSchema);
// else if (myLeftIndexToIndexMap == null && myLeftOp.opSchema != null)
// myLeftIndexToIndexMap = myInitIndexToIndex (myRightOp.opSchema, myLeftOp.opSchema);
// myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
// myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
// }
private int[] myInitIndexToIndex (IBlueTableSchema sch1, IBlueTableSchema sch2)
{
//BlueEventLog.logAssert (sch1.columnNames.Length.Equals (sch2.columnNames.Length));
//foreach (string colName in sch1.columnNames)
// BlueEventLog.logAssert (sch2[colName] != null);
if(!sch1.columnNames.Length.Equals(sch2.columnNames.Length))
throw new BlueDataException("Union stream: different schemas");
foreach(string colName in sch1.columnNames)
{
if(sch2[colName] == null)
throw new BlueDataException("Union stream: can't find " + colName);
}
int[] retVal = new int[sch1.columnNames.Length];
for (int i = 0; i < retVal.Length; i++)
retVal[i] = sch2[sch1[i].colName].colNo;
return retVal;
}
// private int[] myInitIndexToIndex (IBlueTableSchema sch1, IBlueTableSchema sch2)
// {
// //BlueEventLog.logAssert (sch1.columnNames.Length.Equals (sch2.columnNames.Length));
// //foreach (string colName in sch1.columnNames)
// // BlueEventLog.logAssert (sch2[colName] != null);
// if(!sch1.columnNames.Length.Equals(sch2.columnNames.Length))
// throw new BlueDataException("Union stream: different schemas");
// foreach(string colName in sch1.columnNames)
// {
// if(sch2[colName] == null)
// throw new BlueDataException("Union stream: can't find " + colName);
// }
// int[] retVal = new int[sch1.columnNames.Length];
// for (int i = 0; i < retVal.Length; i++)
// retVal[i] = sch2[sch1[i].colName].colNo;
// return retVal;
// }
public virtual void endSession ()
{
myFireEvent (myLastFiredPacket.Clone ());
}
// public virtual void endSession ()
// {
// myFireEvent (myLastFiredPacket.Clone ());
// }
public abstract void add (IBlueRowCollection rows, bool isLeft);
// public abstract void add (IBlueRowCollection rows, bool isLeft);
public abstract void delete (IBlueRowCollection rows, bool isLeft);
// public abstract void delete (IBlueRowCollection rows, bool isLeft);
public abstract void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft);
// public abstract void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft);
private void myInit ()
{
myLastFiredPacket = null;
myLeftOp.init ();
myRightOp.init ();
myLeftIndex.Clear ();
myRightIndex.Clear ();
myIdCounter = myIdIndex = 0;
myLeftIndexToIndexMap = myRightIndexToIndexMap = null;
}
// private void myInit ()
// {
// myLastFiredPacket = null;
// myLeftOp.init ();
// myRightOp.init ();
// myLeftIndex.Clear ();
// myRightIndex.Clear ();
// myIdCounter = myIdIndex = 0;
// myLeftIndexToIndexMap = myRightIndexToIndexMap = null;
// }
/// <summary>
/// Used for test purposes only. Users of this class specifically, should ignore it
/// (too bad, the NUnit cant see the private methods)
/// </summary>
/// <param name="dp"></param>
/// <param name="isLeft"></param>
internal void receiveStream (DataPacket dp, bool isLeft)
{
if (isLeft)
myLeftOp.myChangeHandler (dp);
else
myRightOp.myChangeHandler (dp);
}
public void receiveStreamTest(DataPacket dp, bool isLeft)
{
receiveStream(dp, isLeft);
}
}
}
// /// <summary>
// /// Used for test purposes only. Users of this class specifically, should ignore it
// /// (too bad, the NUnit cant see the private methods)
// /// </summary>
// /// <param name="dp"></param>
// /// <param name="isLeft"></param>
// internal void receiveStream (DataPacket dp, bool isLeft)
// {
// if (isLeft)
// myLeftOp.myChangeHandler (dp);
// else
// myRightOp.myChangeHandler (dp);
// }
// public void receiveStreamTest(DataPacket dp, bool isLeft)
// {
// receiveStream(dp, isLeft);
// }
// }
//}
@@ -114,6 +114,15 @@ namespace BCG {
rv.Add(r.get<T>(col));
return rv;
}
public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, int start_idx, int n_items,string col) {
List<T> rv = new List<T>();
int end_idx = Math.Min(start_idx + n_items, rows.Count());
for (int i = start_idx; i < end_idx; i++) {
var r = rows.ElementAt(i);
rv.Add(r.get<T>(col));
}
return rv;
}
public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col, int start_index, int count) {
List<T> rv = new List<T>();
int i = 0;
@@ -147,7 +147,7 @@ namespace bts.BlueOrb {
}
}
[DataStreamUserRespresentation("FutureHistoricalBars")]
[DataStreamUserRespresentation("FutureHistoricalBarsStream")]
public class FutureHistoricalBarsStream : ViewStream {
public FutureHistoricalBarsStream(string fut_contract, string range_beg, string range_end) {
DateTime dt_range_beg = DateTime.Parse(range_beg);
@@ -1,8 +1,8 @@
using bts.SilverPine;
namespace bts.SilverPine {
using bts.api;
using bts.spark;
namespace bts.BlueOrb {
//using bts.api;
//using bts.spark;
using System;
@@ -16,7 +16,7 @@ namespace bts.SilverPine {
public string contract_text;
}
public class FutureSpreadContract {
public List<Tuple<double, bts.SilverPine.ContractUtils.FutureTradableContract>> legs;
public List<Tuple<double, ContractUtils.FutureTradableContract>> legs;
public string symbol {
get {
return legs == null || legs.Count == 0 ? String.Empty : legs.ElementAt(0).Item2.symbol;
@@ -206,7 +206,7 @@ namespace bts.SilverPine {
}
// tehre are some non calendar spreads we cant parse
//Log.assert(false);
return bts.SilverPine.ContractUtils.ContractType.FutureSpread;
return ContractUtils.ContractType.FutureSpread;
}
public static ContractCategory get_category(string symbol) {
@@ -1,23 +1,23 @@
namespace bts.SilverPine {
using BCG;
//namespace bts.BlueOrb {
// using BCG;
// Finds the regions around specified events
[DataStreamUserRespresentation("EventsPriceAction")]
public class EventsPriceActionStream : ViewStream {
public EventsPriceActionStream(string symbol, string event_type, string pre_event_period,
string post_event_period, int n_events) :
this(symbol, event_type, TimeBasedFilter.parse_duration(pre_event_period), TimeBasedFilter.parse_duration(post_event_period),
n_events, ViewStreamDatabase.instance) { }
// // Finds the regions around specified events
// [DataStreamUserRespresentation("EventsPriceAction")]
// public class EventsPriceActionStream : ViewStream {
// public EventsPriceActionStream(string symbol, string event_type, string pre_event_period,
// string post_event_period, int n_events) :
// this(symbol, event_type, TimeBasedFilter.parse_duration(pre_event_period), TimeBasedFilter.parse_duration(post_event_period),
// n_events, ViewStreamDatabase.instance) { }
public EventsPriceActionStream(string symbol, string event_type, TimeSpan pre_event_period,
TimeSpan post_event_period, int n_events, ViewStreamDatabase vsd) {
// TODO: figure out which bars to use, 1 min or 1 day or live, based on pre_event_period / post_event_period
FuturesActiveContractStream olhcv_str = new FuturesActiveContractStream(symbol);
ViewStream events_str = new EventsStream(vsd, event_type);
// todo
//events_str = new TopStream(events_str, n_events, "-ts");
ViewStream rs = new TimeProximityJoinStream(olhcv_str, "ts_event", events_str, "ts", pre_event_period, post_event_period);
mySetRealSource(rs);
}
}
}
// public EventsPriceActionStream(string symbol, string event_type, TimeSpan pre_event_period,
// TimeSpan post_event_period, int n_events, ViewStreamDatabase vsd) {
// // TODO: figure out which bars to use, 1 min or 1 day or live, based on pre_event_period / post_event_period
// FuturesActiveContractStream olhcv_str = new FuturesActiveContractStream(symbol);
// ViewStream events_str = new EventsStream(vsd, event_type);
// // todo
// //events_str = new TopStream(events_str, n_events, "-ts");
// ViewStream rs = new TimeProximityJoinStream(olhcv_str, "ts_event", events_str, "ts", pre_event_period, post_event_period);
// mySetRealSource(rs);
// }
// }
//}
@@ -1,20 +1,20 @@
namespace bts.SilverPine {
using BCG;
//namespace bts.SilverPine {
// using BCG;
public class IndicatorStream : ViewStream {
enum Indicator {
StandardDeviation = 0
}
public class IndicatorInfo {
Indicator indicator;
IDictionary<string, object> parameters;
}
public IndicatorStream(string symbol_name, string indicator_name, string indicator_params_json) {
// public class IndicatorStream : ViewStream {
// enum Indicator {
// StandardDeviation = 0
// }
// public class IndicatorInfo {
// Indicator indicator;
// IDictionary<string, object> parameters;
// }
// public IndicatorStream(string symbol_name, string indicator_name, string indicator_params_json) {
}
public IndicatorStream(string symbol_name, IndicatorInfo info) {
// }
// public IndicatorStream(string symbol_name, IndicatorInfo info) {
}
// }
}
}
// }
//}
@@ -1,57 +1,57 @@
namespace bts.SilverPine {
using BCG;
//namespace bts.SilverPine {
// using BCG;
using ZstdSharp.Unsafe;
// using ZstdSharp.Unsafe;
[DataStreamUserRespresentation("AbsVolume")]
// Calculates the volume within a time window (this does not do rates, like highest daily volume etc,
// but with some thought can perhaps be extended)
public class AbsoluteVolumeStream : ViewStream {
// TODO, abstract this out
// adds contract_type to OHLCV stream
private static ViewStream _get_ohlcv_steam(string symbol, DateTime d1, DateTime d2) {
var ohlcv_params = OHLCVStream.get_params(d1, d2);
ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
Func<IBlueTableSchema, IBlueTableSchema> create_schema =
s => BlueDataUtility.add_columns(s, Tuple.Create(typeof(string), "contract_type"));
rs = new SuperMapStream(rs,
s => create_schema(s),
r => {
IBlueTableSchema out_schema = create_schema(r.schema);
string contract_txt = r.get<string>("contract");
var contr_type = ContractUtils.get_type(contract_txt);
MutableBlueRow out_row = new MutableBlueRow(out_schema, r);
out_row.setValue("contract_type", contr_type.ToString());
return new IBlueRow[] { out_row };
});
return rs;
}
// [DataStreamUserRespresentation("AbsVolume")]
// // Calculates the volume within a time window (this does not do rates, like highest daily volume etc,
// // but with some thought can perhaps be extended)
// public class AbsoluteVolumeStream : ViewStream {
// // TODO, abstract this out
// // adds contract_type to OHLCV stream
// private static ViewStream _get_ohlcv_steam(string symbol, DateTime d1, DateTime d2) {
// var ohlcv_params = OHLCVStream.get_params(d1, d2);
// ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
// Func<IBlueTableSchema, IBlueTableSchema> create_schema =
// s => BlueDataUtility.add_columns(s, Tuple.Create(typeof(string), "contract_type"));
// rs = new SuperMapStream(rs,
// s => create_schema(s),
// r => {
// IBlueTableSchema out_schema = create_schema(r.schema);
// string contract_txt = r.get<string>("contract");
// var contr_type = ContractUtils.get_type(contract_txt);
// MutableBlueRow out_row = new MutableBlueRow(out_schema, r);
// out_row.setValue("contract_type", contr_type.ToString());
// return new IBlueRow[] { out_row };
// });
// return rs;
// }
public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1)
: this(symbol, ctype, d1, d1) { }
// public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1)
// : this(symbol, ctype, d1, d1) { }
public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1, DateTime d2)
: this(symbol, Enum.Parse<ContractUtils.ContractType>(ctype), d1, d2) {
}
// public AbsoluteVolumeStream(string symbol, string ctype, DateTime d1, DateTime d2)
// : this(symbol, Enum.Parse<ContractUtils.ContractType>(ctype), d1, d2) {
// }
static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
if (dt1 < dt2) {
DateTime aux = dt1;
dt1 = dt2;
dt2 = aux;
}
}
// static void swap_if_necessary(ref DateTime dt1, ref DateTime dt2) {
// if (dt1 < dt2) {
// DateTime aux = dt1;
// dt1 = dt2;
// dt2 = aux;
// }
// }
private AbsoluteVolumeStream(string symbol, ContractUtils.ContractType ctype, DateTime d1, DateTime d2) {
swap_if_necessary(ref d1, ref d2);
ViewStream ohlcv_str = _get_ohlcv_steam(symbol, d1, d2);
ohlcv_str = new FilterStream(ohlcv_str,
r => r.get_datetime("ts_event") <= d1 && r.get_datetime("ts_event") >= d2);
ohlcv_str = new FilterStream(ohlcv_str, r => String.Compare(ctype.ToString(), r.get<string>("contract_type"), true) == 0);
ViewStream rs = new MaxStream(ohlcv_str, "volume");
rs = new NewProjectStream(rs, "volume:total_volume contract !");
mySetRealSource(rs);
}
// private AbsoluteVolumeStream(string symbol, ContractUtils.ContractType ctype, DateTime d1, DateTime d2) {
// swap_if_necessary(ref d1, ref d2);
// ViewStream ohlcv_str = _get_ohlcv_steam(symbol, d1, d2);
// ohlcv_str = new FilterStream(ohlcv_str,
// r => r.get_datetime("ts_event") <= d1 && r.get_datetime("ts_event") >= d2);
// ohlcv_str = new FilterStream(ohlcv_str, r => String.Compare(ctype.ToString(), r.get<string>("contract_type"), true) == 0);
// ViewStream rs = new MaxStream(ohlcv_str, "volume");
// rs = new NewProjectStream(rs, "volume:total_volume contract !");
// mySetRealSource(rs);
// }
}
}
// }
//}
@@ -1,4 +1,4 @@
namespace bts.SilverPine {
namespace bts.BlueOrb {
using BCG;
using SharpCompress.Common.Rar;
@@ -1,4 +1,4 @@
namespace bts.SilverPine {
namespace bts.BlueOrb {
using BCG;
using bts.api;
@@ -1,4 +1,4 @@
namespace bts.SilverPine {
namespace bts.BlueOrb {
using BCG;
using System.Diagnostics;
@@ -1,4 +1,4 @@
namespace bts.SilverPine{
namespace bts.BlueOrb{
using BCG;
public class SpreadStatArbsWeightLinearRegressionStream : ViewStream {
private IBlueTableSchema _schema;
@@ -1,4 +1,4 @@
namespace bts.SilverPine {
namespace bts.BlueOrb {
using BCG;
using MathNet.Numerics.LinearAlgebra.Factorization;
@@ -2,7 +2,7 @@
using System.Collections.Generic;
using MathNet.Numerics.LinearRegression;
namespace bts.SilverPine {
namespace bts.BlueOrb {
public static class SpreadWeightCalculator {
/// <summary>
@@ -1,40 +1,40 @@
namespace bts.SilverPine {
using BCG;
//namespace bts.SilverPine {
// using BCG;
using MathNet.Numerics.Statistics;
// using MathNet.Numerics.Statistics;
// calculates the reazlied standard deviation of a series of prices
public class StandardDeviationStream : ViewStream {
public StandardDeviationStream(string symbol, string std_dev_period)
:
this(symbol, TimeBasedFilter.parse_duration(std_dev_period)) { }
// // calculates the reazlied standard deviation of a series of prices
// public class StandardDeviationStream : ViewStream {
// public StandardDeviationStream(string symbol, string std_dev_period)
// :
// this(symbol, TimeBasedFilter.parse_duration(std_dev_period)) { }
public StandardDeviationStream(string symbol, TimeSpan std_dev_period) {
// since this is a 1_day bar str, 90 rows => 90 days, so no need to filter
int n_days = std_dev_period.Days;
ViewStream ohlcv_str = new FuturesActiveContractStream(symbol);
IBlueTableSchema out_schema = null;
int id = 0;
Func<List<IBlueRow>, uint, IBlueRow> find_std_dev = (rows, idx) => {
int iidx = Convert.ToInt32(idx);
if (rows == null || rows.Count == 0)
return null;
IBlueRow ir = rows.ElementAt(iidx);
out_schema = out_schema ?? ir.schema.add_column(TypeCode.Double, "std_dev");
MutableBlueRow rv = new MutableBlueRow(out_schema, ir);
rv.setValue("std_dev", Double.NaN);
IEnumerable<double> last_n_close_prices = rows.Skip(iidx).Take(n_days).Select(r => r.get<double>("close"));
if (last_n_close_prices.Count() >= n_days) {
double std_dev = MathNet.Numerics.Statistics.Statistics.StandardDeviation(last_n_close_prices);
rv.setValue("std_dev", std_dev);
}
//rv.setValue("id", id++);
return rv;
};
ViewStream rs = new GlobalMapStream(ohlcv_str, "ts_event", find_std_dev);
rs = new NullFilterStream(rs, "std_dev", true);
mySetRealSource(rs);
}
// public StandardDeviationStream(string symbol, TimeSpan std_dev_period) {
// // since this is a 1_day bar str, 90 rows => 90 days, so no need to filter
// int n_days = std_dev_period.Days;
// ViewStream ohlcv_str = new FuturesActiveContractStream(symbol);
// IBlueTableSchema out_schema = null;
// int id = 0;
// Func<List<IBlueRow>, uint, IBlueRow> find_std_dev = (rows, idx) => {
// int iidx = Convert.ToInt32(idx);
// if (rows == null || rows.Count == 0)
// return null;
// IBlueRow ir = rows.ElementAt(iidx);
// out_schema = out_schema ?? ir.schema.add_column(TypeCode.Double, "std_dev");
// MutableBlueRow rv = new MutableBlueRow(out_schema, ir);
// rv.setValue("std_dev", Double.NaN);
// IEnumerable<double> last_n_close_prices = rows.Skip(iidx).Take(n_days).Select(r => r.get<double>("close"));
// if (last_n_close_prices.Count() >= n_days) {
// double std_dev = MathNet.Numerics.Statistics.Statistics.StandardDeviation(last_n_close_prices);
// rv.setValue("std_dev", std_dev);
// }
// //rv.setValue("id", id++);
// return rv;
// };
// ViewStream rs = new GlobalMapStream(ohlcv_str, "ts_event", find_std_dev);
// rs = new NullFilterStream(rs, "std_dev", true);
// mySetRealSource(rs);
// }
}
}
// }
//}
@@ -1,4 +1,4 @@
namespace bts.SilverPine {
namespace bts.BlueOrb {
using BCG.Utilities.General;
using bts.md;
@@ -1,5 +1,6 @@
namespace BCG {
namespace bts.BlueOrb{
using BCG.Utilities.General;
using BCG;
using bts.md;
@@ -1,4 +1,4 @@
namespace bts.SilverPine {
namespace bts.BlueOrb {
using BCG;
[DataStreamUserRespresentation("Events")]
@@ -15,4 +15,4 @@ namespace bts.SilverPine {
mySetRealSource(rs);
}
}
}
}
@@ -1,4 +1,4 @@
namespace bts.SilverPine {
namespace bts.BlueOrb{
using BCG;
using bts;
@@ -1,4 +1,4 @@
namespace bts.SilverPine {
namespace bts.BlueOrb {
using BCG.Utilities.General;
using bts.api;
@@ -1,37 +1,37 @@
namespace bts.SilverPine {
using BCG;
//namespace bts.SilverPine {
// using BCG;
[DataStreamUserRespresentation("OHLCVPriceDiff")]
public class OHLCVPriceDiffStream : ViewStream {
// TODO, abstract this out
private static ViewStream _get_ohlcv_steam(string symbol_or_contract, DateTime d1, DateTime d2) {
var tup = ContractUtils.get_symbol_category(symbol_or_contract);
var ohlcv_params = OHLCVStream.get_params(d1, d2);
var cont_cat = tup.Item1;
string symbol = tup.Item2;
ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
switch (cont_cat) {
case ContractUtils.ContractCategory.FutureString:
case ContractUtils.ContractCategory.SpreadString:
rs = new FilterStream(rs, $"contract = '{symbol_or_contract}'");
break;
case ContractUtils.ContractCategory.Symbol:
rs = new FuturesActiveContractStream(rs);
break;
}
return rs;
}
// [DataStreamUserRespresentation("OHLCVPriceDiff")]
// public class OHLCVPriceDiffStream : ViewStream {
// // TODO, abstract this out
// private static ViewStream _get_ohlcv_steam(string symbol_or_contract, DateTime d1, DateTime d2) {
// var tup = ContractUtils.get_symbol_category(symbol_or_contract);
// var ohlcv_params = OHLCVStream.get_params(d1, d2);
// var cont_cat = tup.Item1;
// string symbol = tup.Item2;
// ViewStream rs = new OHLCVStream(symbol, ohlcv_params);
// switch (cont_cat) {
// case ContractUtils.ContractCategory.FutureString:
// case ContractUtils.ContractCategory.SpreadString:
// rs = new FilterStream(rs, $"contract = '{symbol_or_contract}'");
// break;
// case ContractUtils.ContractCategory.Symbol:
// rs = new FuturesActiveContractStream(rs);
// break;
// }
// return rs;
// }
public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2)
: this(symbol_or_contract, d1, d2, "close") { }
// public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2)
// : this(symbol_or_contract, d1, d2, "close") { }
public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2, string col) {
ViewStream rs = _get_ohlcv_steam(symbol_or_contract, d1, d2);
rs = new ColumnDifferenceStream(rs, r => d1.Equals(r.get_datetime("ts_event")),
r => d2.Equals(r.get_datetime("ts_event")), col, col);
rs = new NewProjectStream(rs, "diff !");
mySetRealSource(rs);
}
// public OHLCVPriceDiffStream(string symbol_or_contract, DateTime d1, DateTime d2, string col) {
// ViewStream rs = _get_ohlcv_steam(symbol_or_contract, d1, d2);
// rs = new ColumnDifferenceStream(rs, r => d1.Equals(r.get_datetime("ts_event")),
// r => d2.Equals(r.get_datetime("ts_event")), col, col);
// rs = new NewProjectStream(rs, "diff !");
// mySetRealSource(rs);
// }
}
}
// }
//}
@@ -1,4 +1,4 @@
namespace bts.SilverPine {
namespace bts.BlueOrb {
using bts;
using BCG;
using MongoDB.Driver;
@@ -1,8 +1,7 @@
using System;
using BCG;
namespace bts.SilverPine
{
namespace bts.BlueOrb {
/*
----------------------------------------------------------------------
*Expression*
@@ -1,32 +1,32 @@
namespace bts.SilverPine {
using BCG;
//namespace bts.SilverPine {
// using BCG;
// this is an internal stream. its job is to take an OHLCV stream
// and filter it for
// 1. symbol (table_name and active contract) and
// 2. time (can be a day of the week, month, and a time of the day period)
public class BarsSearcherStream : ViewStream {
public BarsSearcherStream(string symbol, OHLCVStream.OHLCVBarSize bar_size,
string day_of_week, TimeSpan tod_begin, TimeSpan tod_end) {
ViewStream rs = new OHLCVStream(symbol, bar_size);
rs = new DateTimeExpanstionStream(rs, "ts_event");
rs = new FilterStream(rs, r => {
DateTime ts = r.get_datetime("ts_event");
return r.get<string>("ts_event_day") == day_of_week.ToString() &&
ts.TimeOfDay >= tod_begin && ts.TimeOfDay <= tod_end;
});
// look at symbol and either filter ir ot do the active contracts
// assuming its symbol
rs = new NonIncJoinStream(rs, "ts_event_date", "open high low close volume contract ts_event",
new FuturesActiveContractStream(symbol), "ts_event", "ts_event:date contract:active_contract");
// filter it for the active contract only
rs = new FilterStream(rs, r => String.Compare(r.get<string>("contract"), r.get<string>("active_contract")) == 0);
mySetRealSource(rs);
}
public BarsSearcherStream(string symbol, string bar_size,
string day_of_week, string tod_begin, string tod_end)
:
this(symbol, Enum.Parse<OHLCVStream.OHLCVBarSize>(bar_size), day_of_week,
TimeSpan.Parse(tod_begin), TimeSpan.Parse(tod_end)) { }
}
}
// // this is an internal stream. its job is to take an OHLCV stream
// // and filter it for
// // 1. symbol (table_name and active contract) and
// // 2. time (can be a day of the week, month, and a time of the day period)
// public class BarsSearcherStream : ViewStream {
// public BarsSearcherStream(string symbol, OHLCVStream.OHLCVBarSize bar_size,
// string day_of_week, TimeSpan tod_begin, TimeSpan tod_end) {
// ViewStream rs = new OHLCVStream(symbol, bar_size);
// rs = new DateTimeExpanstionStream(rs, "ts_event");
// rs = new FilterStream(rs, r => {
// DateTime ts = r.get_datetime("ts_event");
// return r.get<string>("ts_event_day") == day_of_week.ToString() &&
// ts.TimeOfDay >= tod_begin && ts.TimeOfDay <= tod_end;
// });
// // look at symbol and either filter ir ot do the active contracts
// // assuming its symbol
// rs = new NonIncJoinStream(rs, "ts_event_date", "open high low close volume contract ts_event",
// new FuturesActiveContractStream(symbol), "ts_event", "ts_event:date contract:active_contract");
// // filter it for the active contract only
// rs = new FilterStream(rs, r => String.Compare(r.get<string>("contract"), r.get<string>("active_contract")) == 0);
// mySetRealSource(rs);
// }
// public BarsSearcherStream(string symbol, string bar_size,
// string day_of_week, string tod_begin, string tod_end)
// :
// this(symbol, Enum.Parse<OHLCVStream.OHLCVBarSize>(bar_size), day_of_week,
// TimeSpan.Parse(tod_begin), TimeSpan.Parse(tod_end)) { }
// }
//}
@@ -2,34 +2,38 @@ namespace bts.SilverPine {
using BCG;
using bts;
using bts.SilverPine;
//using bts.SilverPine;
[DataStreamUserRespresentation("nl")]
public class NLStream : ViewStream {
private readonly string _nl_prompt;
private ViewStream _nl_to_dsl_stream, _dsl_stream;
public NLStream(string nl_prompt) {
_nl_prompt = nl_prompt;
}
namespace bts.BlueOrb {
protected override void myOnFirstSubscriber() {
_nl_to_dsl_stream = new NLToDSLStream(_nl_prompt);
_nl_to_dsl_stream.onChange += _handle_dsl_callback;
}
[DataStreamUserRespresentation("nl")]
public class NLStream : ViewStream {
private readonly string _nl_prompt;
private ViewStream _nl_to_dsl_stream, _dsl_stream;
public NLStream(string nl_prompt) {
_nl_prompt = nl_prompt;
}
private void _handle_dsl_callback(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
var r = dp.newRows.to_list().ElementAt(0);
string dsl = r.get<string>("response");
Log.Gen.info($"NLStream._handle_dsl_callback dsl: {dsl}");
_dsl_stream = new DSLStream(dsl);
_dsl_stream.onChange += _handle_data_cb;
}
protected override void myOnFirstSubscriber() {
_nl_to_dsl_stream = new NLToDSLStream(_nl_prompt);
_nl_to_dsl_stream.onChange += _handle_dsl_callback;
}
private void _handle_data_cb(object dp_obj) {
var dp = dp_obj as DataPacket;
DataPacket dp_out = BlueDataUtility.build_dp_inc(dp.newRows, myLastValue as DataPacket);
myFireEvent(dp_out);
private void _handle_dsl_callback(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
var r = dp.newRows.to_list().ElementAt(0);
string dsl = r.get<string>("response");
Log.Gen.info($"NLStream._handle_dsl_callback dsl: {dsl}");
_dsl_stream = new DSLStream(dsl);
_dsl_stream.onChange += _handle_data_cb;
}
private void _handle_data_cb(object dp_obj) {
var dp = dp_obj as DataPacket;
DataPacket dp_out = BlueDataUtility.build_dp_inc(dp.newRows, myLastValue as DataPacket);
myFireEvent(dp_out);
}
}
}
}
}
@@ -169,6 +169,12 @@ namespace mbts.analytics {
return 0;
}
}
// if < 0 then rebate, > 0 we pay a fee
public double trade_transaction_fee{
get{
}
}
public double pos_gamma {
get {
return tl_row.security_type == SecurityType.Option ?