diff --git a/externals b/externals
index 8069cc893..ff5f02bbd 160000
--- a/externals
+++ b/externals
@@ -1 +1 @@
-Subproject commit 8069cc893f5cef3b6cb0e989bbdf6514cc153d29
+Subproject commit ff5f02bbde0f584493835ac267af34b31e637845
diff --git a/src/DataStreamSuite/DataStream/DataStream.csproj b/src/DataStreamSuite/DataStream/DataStream.csproj
index 8c8ed8d6b..b929774ae 100644
--- a/src/DataStreamSuite/DataStream/DataStream.csproj
+++ b/src/DataStreamSuite/DataStream/DataStream.csproj
@@ -18,8 +18,10 @@
+
+
diff --git a/src/DataStreamSuite/DataStream/Misc/DSLStream.cs b/src/DataStreamSuite/DataStream/Misc/DSLStream.cs
index 5824a60e1..c652e15e8 100644
--- a/src/DataStreamSuite/DataStream/Misc/DSLStream.cs
+++ b/src/DataStreamSuite/DataStream/Misc/DSLStream.cs
@@ -13,6 +13,7 @@ namespace bts.SilverPine {
Log.Gen.info($"DSLStream.cons dsl: {dsl}");
_dsl = dsl;
ViewStream rs = build_stream(dsl);
+ rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
mySetRealSource(rs);
}
@@ -23,7 +24,7 @@ namespace bts.SilverPine {
return rv;
} catch (Exception ex) {
- bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
+ Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
return null;
}
}
diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/NullFilterStream.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/NullFilterStream.cs
new file mode 100644
index 000000000..62feaf799
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Primitives/Filter/NullFilterStream.cs
@@ -0,0 +1,22 @@
+namespace bts.SilverPine {
+ using BCG;
+
+ using System;
+
+ public class NullFilterStream : ViewStream {
+ public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
+ ViewStream rs = new FilterStream(parent_str,
+ r => {
+ bool is_dbnull = r.is_null(col_name);
+ if (is_dbnull) {
+ return false;
+ }
+ if (r.schema[col_name].type_code() == TypeCode.Double) {
+ return !Double.IsNaN(r.get(col_name));
+ }
+ return true;
+ });
+ mySetRealSource(rs);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs
index adea44bac..092152d22 100644
--- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs
@@ -9,6 +9,10 @@ using mbts;
namespace BCG {
public class NonIncJoinSpec {
public ViewStream stream;
+ // join col names are the names on which the streams are joined (for equality),
+ // data col names are the names in the original stream
+ // projected col naes are their new names, ie if its x:l_x, then x is in the
+ // data_col_names and l_x in projected cols
public string[] join_col_names, data_col_names, projected_col_names;
public NonIncJoinSpec() { }
diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs
index 07b9f6528..92232eb71 100644
--- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs
@@ -124,11 +124,12 @@ namespace BCG {
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
timer.Stop();
_n_total_fires++;
- if (timer.ElapsedMilliseconds >= 250) {
- _n_expensive_fires++;
- string log_msg = $"Join and map time for {_book_keeping_data.Count} "
- + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
- }
+ //if (timer.ElapsedMilliseconds >= 250) {
+ _n_expensive_fires++;
+ string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+ + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
+ Log.Gen.info(log_msg);
+ //}
var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
myFireEvent(dp_out);
//myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/Proximity/ProximityJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/Proximity/ProximityJoinStream.cs
new file mode 100644
index 000000000..aeb26028b
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Primitives/Join/Proximity/ProximityJoinStream.cs
@@ -0,0 +1,68 @@
+namespace bts {
+ using BCG;
+ using BCG.Utilities.General;
+
+ using System;
+ using System.Diagnostics;
+
+ using ZstdSharp.Unsafe;
+
+ // functon that joins the 2 rows and returns the outputput row. The function
+ // can return a null which means the 2 rows dont need to be joined
+ using JoinFunc = System.Func;
+ public class ProximityJoinStream : ViewStream, IWorker {
+ private ViewStream _lvs, _rvs;
+ private JoinFunc _join_func;
+ private bool _dirty = false;
+ private Stopwatch _stopwatch = new Stopwatch();
+ private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
+ public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
+ _lvs = lvs;
+ _rvs = rvs;
+ _join_func = join_func;
+ }
+
+ protected override void myOnFirstSubscriber() {
+ _lvs.onChange += _handle_data;
+ _rvs.onChange += _handle_data;
+ _vsd.addWorker(this);
+ }
+
+ protected override void myOnLastSubscriber() {
+ _lvs.onChange -= _handle_data;
+ _rvs.onChange -= _handle_data;
+ _vsd.removeWorker(this);
+ }
+
+ private void _handle_data(object dp_obj) {
+ _dirty = true;
+ doWork();
+ }
+
+ public void doWork() {
+ Func all_data_revcd = () => {
+ return _lvs.rows != null && _rvs.rows != null;
+ };
+ _stopwatch.Start();
+ if (_dirty && all_data_revcd()) {
+ _dirty = false;
+ BlueTable rv = null;
+ foreach (IBlueRow lr in _lvs.rows) {
+ foreach (IBlueRow rr in _rvs.rows) {
+ IBlueRow resr = _join_func(lr, rr);
+ if (resr == null)
+ continue;
+ rv = rv ?? new BlueTable(resr.schema);
+ rv.addRow(resr);
+ }
+ }
+ _stopwatch.Stop();
+ Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
+ var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
+ myFireEvent(dp);
+ }
+
+ }
+
+ }
+}
\ No newline at end of file
diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/Proximity/TimeProximityJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/Proximity/TimeProximityJoinStream.cs
new file mode 100644
index 000000000..0f48e5265
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Primitives/Join/Proximity/TimeProximityJoinStream.cs
@@ -0,0 +1,47 @@
+namespace bts.SilverPine {
+ using BCG;
+
+ using System;
+ using System.CodeDom;
+ using System.Configuration;
+ using System.Reactive.Disposables;
+
+ public class TimeProximityJoinStream : ViewStream {
+ private int _id = 0;
+
+ public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
+ string pre_join_period, string post_join_period) :
+ this(ls, l_time_col, rs, r_time_col, TimeBasedFilter.parse_duration(pre_join_period),
+ TimeBasedFilter.parse_duration(post_join_period)) { }
+
+
+ public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
+ TimeSpan pre_join_period, TimeSpan post_join_period) {
+ Func create_row = (lr, rr) => {
+ var id = 0;
+ lock (this) {
+ id = _id++;
+ }
+ IBlueRow rv = BlueDataUtility.create_composite_row(lr, rr, id);
+ return rv;
+ };
+ Func join_func = (lr, rr) => {
+ DateTime ldt = lr.get_datetime(l_time_col);
+ DateTime rdt = rr.get_datetime(r_time_col);
+ TimeSpan diff = ldt - rdt;
+ if (ldt < rdt) {
+ if (diff.Duration() > pre_join_period) {
+ return null;
+ }
+ } else {
+ if (diff > post_join_period)
+ return null;
+ }
+ return create_row(lr, rr);
+ };
+
+ ViewStream rstr = new ProximityJoinStream(ls, rs, join_func);
+ mySetRealSource(rstr);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs b/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs
index 9fe3d6198..05766e90c 100644
--- a/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs
@@ -26,7 +26,7 @@ namespace BCG {
return rv;
}
- //literal("table1", "x:123 y:1.001 z:abc;x:123 y:1.001 z:abc")
+ //literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
public LiteralStream(string table_name, string rows) {
Func get_row = (json_row, id) => {
Dictionary map = new Dictionary();
diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs b/src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs
index 3b4e81539..098350ddf 100644
--- a/src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs
@@ -137,10 +137,24 @@ namespace bts {
HashSet removed_cols = new HashSet();
Dictionary mapped_to_orig_map = new Dictionary();
Dictionary orig_to_mapped = new Dictionary();
+ // by default every col will show up, so if we have the projection as (a:a1)
+ // it will convert a to a1,
+ // examples: if a schema has cols a b c
+ // project(parent_str, "a:aa ") -> aa b c
+ // project(parent_str, "a:aa !c") -> aa b
+ // project(parent_str, "a:aa !") -> aa
+ // project(parent_str, "a:aa b = (aa*aa)!") -> aa b
+ bool hide_all = false;
foreach (string p_tok in project_toks) {
if (p_tok.StartsWith("!")) {
- string removed_col = p_tok.Substring(1);
- removed_cols.Add(removed_col);
+ if (p_tok.Length == 1) {
+ // if it is a "blanket "!"=> hide all cols except the one that
+ // are stated explicitly
+ hide_all = true;
+ } else {
+ string removed_col = p_tok.Substring(1);
+ removed_cols.Add(removed_col);
+ }
} else if (p_tok.Contains(":")) {
string[] map_toks = p_tok.Split(":");
string orig_col = map_toks[0];
@@ -155,12 +169,11 @@ namespace bts {
}
}
}
+
// make the new schema
List new_cols = new List();
foreach (ColumnInfo ci in input_schema.columnInfo) {
if (ci.colName == "id") {
- // if there's a hidden col, other columns will show by default, but if cols are being
- // projected epxlicitely, than others wil be hidden
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
new_cols.Add(nc);
continue;
@@ -172,12 +185,13 @@ namespace bts {
ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
new_cols.Add(nc);
} else {
- if (ci.colName == "id" || removed_cols.Count > 0) {
- // if there's a hidden col, other columns will show by default, but if cols are being
- // projected epxlicitely, than others wil be hidden
- ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
- new_cols.Add(nc);
- }
+ if (hide_all)
+ continue;
+ //if (ci.colName == "id" || removed_cols.Count > 0) {
+ // unless columns are explicitely hidden, all columns show up
+ ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+ new_cols.Add(nc);
+ //}
}
}
IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
diff --git a/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs
index 407ee4c24..818f3e60c 100644
--- a/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs
@@ -1,10 +1,167 @@
using System;
using System.Data;
using System.Data.SQLite;
+using System.Diagnostics;
+using System.Collections;
+
using BCG;
+using System.Linq;
+using System.Data.Entity;
+using System.Collections.Generic;
+
+namespace bts.SilverPine {
+
+ public class DataRowWrapperRow : IBlueRow {
+ private readonly DataRow _row;
+ private readonly IBlueTableSchema _schema;
+ private readonly int _id;
+ public DataRowWrapperRow(IBlueTableSchema schema, DataRow row, int id) {
+ _row = row;
+ _id = id;
+ _schema = schema;
+ }
+ public virtual IBlueTableSchema schema { get { return _schema; } }
+
+ public virtual object this[string col_name] {
+ get {
+ ColumnInfo ci = _schema[col_name];
+ return this[ci.colNo];
+ }
+ }
+ public virtual object this[int index] {
+ get {
+ if (_schema[index].colName == "id")
+ return _id;
+ return _row[index];
+ }
+ }
+
+ public virtual int Count { get { return _row.ItemArray.Length; } }
+
+ public virtual bool IsSynchronized { get { return false; } }
+
+ public virtual object SyncRoot { get { return this; } }
+
+ public virtual void CopyTo(Array array, int index) { _row.ItemArray.CopyTo(array, index); }
+
+ public virtual bool IsFixedSize { get { return true; } }
+
+ virtual public bool IsReadOnly { get { return true; } }
+
+ ///
+ /// The key can either be the string corrsponding to the column name or
+ /// the order of the column
+ ///
+ public virtual object this[object key] {
+ get {
+ if (key is string)
+ return this[(string)key];
+ else if (key is int)
+ return this[(int)key];
+ throw new BlueDataException("Wrong type while indexing the row");
+ }
+ set { throw new NotSupportedException("The IBlueRow is readonly"); }
+ }
+
+ public virtual ICollection Keys { get { return _schema.columnNames; } }
+
+ public virtual ICollection Values { get { return _row.ItemArray; } }
+
+ public virtual void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+ public virtual void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+ public virtual bool Contains(object key) { return _schema[(string)key] != null; }
+
+ public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
+
+ IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
+
+ public virtual void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
+
+ }
+
+ public class DataTableWrapperTable : IBlueRowCollection {
+
+ public class DataTableWrapperTableEnumerator : IEnumerator {
+ private IEnumerator _enumerator;
+ private IBlueTableSchema _schema;
+ private int _id = 0;
+ public DataTableWrapperTableEnumerator(DataTable table) {
+ _enumerator = table.Rows.GetEnumerator();
+ _schema = BlueDataUtility.getSchema(table);
+
+ }
+ public object Current {
+ get {
+ DataRow r = _enumerator.Current as DataRow;
+ return new DataRowWrapperRow(_schema, r, _id);
+ }
+ }
+ public bool MoveNext() {
+ _id++;
+ return _enumerator.MoveNext();
+ }
+ public void Reset() {
+ _id = 0;
+ _enumerator.Reset();
+ }
+ }
+
+ private readonly DataTable _table;
+ private IBlueTableSchema _schema;
+ private IEnumerator _enumerator;
+
+ public DataTableWrapperTable(DataTable table) {
+ _table = table;
+ _schema = BlueDataUtility.getSchema(_table);
+ _enumerator = new DataTableWrapperTableEnumerator(table);
+ }
+ public virtual IBlueRow this[int id] {
+ get {
+ // its assumed the rows index == id
+ DataRow dr = _table.Rows[id];
+ return new DataRowWrapperRow(_schema, dr, id);
+ }
+ set {
+
+ }
+ }
+
+ public virtual IBlueTableSchema schema { get { return _schema; } }
+
+ public virtual bool containsID(int id) { return id < _table.Rows.Count; }
+
+ // private List _rows() {
+ // List rv = new List();
+ // int id = 0;
+ // foreach (DataRow r in _table.Rows) {
+ // rv.Add(new DataRowWrapperRow(r, id++));
+ // }
+ // return rv;
+ // }
+
+ public virtual IEnumerator GetEnumerator() {
+ // var rows = _rows();
+ // return rows.GetEnumerator();
+ return _enumerator;
+ }
+
+ public virtual bool IsSynchronized { get { return false; } }
+
+ public virtual object SyncRoot { get { return this; } }
+
+ public virtual void CopyTo(Array array, int index) {
+ Log.assert(false);
+ //_rows().CopyTo(array as IBlueRow[], index);
+ }
+
+ public virtual int Count {
+ get { return _table.Rows.Count; }
+ }
+ }
-namespace bts {
public static class SQLiteHelper {
public static DataTable ExecuteSelectQuery(string connection_string, string query) {
// Create a new DataTable to store the results
@@ -35,6 +192,7 @@ namespace bts {
[DataStreamUserRespresentation("SQLite")]
public class SQLiteStream : ViewStream {
private readonly string _connection_string, _query;
+ private Stopwatch _stopwatch = new Stopwatch();
public SQLiteStream(string connection_string, string query) {
_connection_string = connection_string;
_query = query;
@@ -42,9 +200,15 @@ namespace bts {
private void _read_from_sqlite_nd_fire() {
+ _stopwatch.Start();
DataTable dt = SQLiteHelper.ExecuteSelectQuery(_connection_string, _query);
+ _stopwatch.Stop();
+ Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
+ _stopwatch.Start();
IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
-
+ //IBlueRowCollection rows = new DataTableWrapperTable(dt);
+ _stopwatch.Stop();
+ Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
myFireEvent(dp);
}
@@ -63,3 +227,4 @@ namespace bts {
}
}
+
diff --git a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs
index 5991a25e0..41eb76390 100644
--- a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs
@@ -3,12 +3,15 @@ using System.Collections.Generic;
using System.Text;
using System.IO;
using BCG.Utilities;
+using System.Diagnostics;
+using Microsoft.Data.Analysis;
+using Parquet;
+using Parquet.Data;
-namespace BCG
-{
+
+namespace BCG {
[DataStreamUserRespresentation("CSVSaver")]
- public class CSVSaverStream : ViewStream, ISideEffectStream
- {
+ public class CSVSaverStream : ViewStream, ISideEffectStream {
private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
new ColumnInfo("fileName", 1, typeof(string)),
new ColumnInfo("nWrites", 2, typeof(int)),
@@ -20,11 +23,9 @@ namespace BCG
private int myNWrites = 0, myNMaxWrites = Int32.MaxValue;
public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName)
- : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false)
- { }
+ : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) { }
- public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot)
- {
+ public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) {
myStream = vs;
myFullyQualifiedFileName = fullyQualifiedFileName;
myOverwrite = overwrite;
@@ -36,18 +37,14 @@ namespace BCG
protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
- private void myHandleChanges(object newValue)
- {
+ private void myHandleChanges(object newValue) {
IBlueRowCollection totalState = (newValue as DataPacket).newRows;
- if (totalState.Count > 0 && myNWrites < myNMaxWrites)
- {
+ if (totalState.Count > 0 && myNWrites < myNMaxWrites) {
myNWrites++;
int i = 0;
StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write));
- foreach (IBlueRow r in totalState)
- {
- if (i++ == 0)
- {
+ foreach (IBlueRow r in totalState) {
+ if (i++ == 0) {
string header = String.Empty;
foreach (string col in r.schema.columnNames)
if (col != "id")
@@ -59,20 +56,17 @@ namespace BCG
writer.Flush();
}
writer.Close();
- if (myFireResultSnapshot)
- {
+ if (myFireResultSnapshot) {
IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
BlueTable nRows = new BlueTable(myResultSchema);
nRows.addRow(nr);
myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
- }
- else
+ } else
myFireEvent(newValue);
}
}
- private string myGetNewFileName()
- {
+ private string myGetNewFileName() {
string[] tags = myFullyQualifiedFileName.Split('.');
if (myOverwrite)
return tags[0] + ".csv";
@@ -80,14 +74,11 @@ namespace BCG
return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv";
}
- private static string myGetCSVRow(IBlueRow r)
- {
+ private static string myGetCSVRow(IBlueRow r) {
string rv = String.Empty;
int i = 0;
- foreach (string col in r.schema.columnNames)
- {
- if (col != "id")
- {
+ foreach (string col in r.schema.columnNames) {
+ if (col != "id") {
string colText = String.Empty;
if (r[col] != null && r[col] != DBNull.Value)
colText = r[col].ToString();
@@ -99,8 +90,7 @@ namespace BCG
return rv;
}
- private string myFormateDateTime(DateTime dt)
- {
+ private string myFormateDateTime(DateTime dt) {
return
string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}",
dt.Year.ToString("0000"),
@@ -114,52 +104,49 @@ namespace BCG
}
[DataStreamUserRespresentation("CSVReader")]
- public class CSVReaderStream : ViewStream
- {
- private IBlueRowCollection myRows;
-
- public CSVReaderStream(string fileAddress)
- {
- myRows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(fileAddress);
+ public class CSVReaderStream : ViewStream {
+ private readonly string _file_address;
+ Stopwatch _stopwatch = new Stopwatch();
+ public CSVReaderStream(string fileAddress) {
+ _file_address = fileAddress;
}
- protected override void myOnFirstSubscriber()
- {
- myFireEvent(new DataPacket(myRows, myRows, new BlueTable(myRows.schema)));
+ protected override void myOnFirstSubscriber() {
+ _stopwatch.Start();
+ DataFrame df = DataFrame.LoadCsv(_file_address);
+ _stopwatch.Stop();
+ bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
+ _stopwatch.Start();
+ IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address);
+ _stopwatch.Stop();
+ bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
+ DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
+ myFireEvent(dp_out);
}
}
- [DataStreamUserRespresentation ("CSVReaderStreamMapper")]
- public class CSVReaderStreamMapperStream : ViewStream
- {
- public CSVReaderStreamMapperStream(ViewStream stream, string output_schema)
- {
+ [DataStreamUserRespresentation("CSVReaderStreamMapper")]
+ public class CSVReaderStreamMapperStream : ViewStream {
+ public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) {
mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema)));
}
}
- public class CSVReaderStreamMapper : IStreamMapper
- {
+ public class CSVReaderStreamMapper : IStreamMapper {
private IBlueTableSchema mySchema;
private readonly string[] myOutputSchemaStringArray;
- public CSVReaderStreamMapper(string outputSchema)
- {
+ public CSVReaderStreamMapper(string outputSchema) {
myOutputSchemaStringArray = outputSchema.Split(' ');
}
- public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
- {
- if (mySchema == null)
- {
+ public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
+ if (mySchema == null) {
ColumnInfo[] ci = inputSchema.columnInfo;
- foreach (string col in myOutputSchemaStringArray)
- {
+ foreach (string col in myOutputSchemaStringArray) {
string colName = col.Substring(0, col.IndexOf('\\')).ToLower();
- for (int i = 0; i < ci.Length; i++)
- {
- if (ci[i].colName.ToLower().Equals(colName))
- {
+ for (int i = 0; i < ci.Length; i++) {
+ if (ci[i].colName.ToLower().Equals(colName)) {
Type t = myGetType(col.Substring(col.IndexOf('\\') + 1));
ci[i] = new ColumnInfo(ci[i].colName, i, t);
break;
@@ -171,74 +158,52 @@ namespace BCG
return mySchema;
}
- public IBlueRow map(IBlueRow row)
- {
+ public IBlueRow map(IBlueRow row) {
object[] vals = new object[row.schema.columnInfo.Length];
row.Values.CopyTo(vals, 0);
- for (int i = 0; i < mySchema.columnInfo.Length; i++)
- {
+ for (int i = 0; i < mySchema.columnInfo.Length; i++) {
Type t = mySchema.columnInfo[i].colType;
- try
- {
+ try {
vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture);
- }
- catch
- {
- if (t.Equals(typeof(Int16)))
- {
+ } catch {
+ if (t.Equals(typeof(Int16))) {
Int16 result;
- if (Int16.TryParse((string)vals[i], out result))
- {
+ if (Int16.TryParse((string)vals[i], out result)) {
vals[i] = result;
continue;
}
double r2;
- if (Double.TryParse((string)vals[i], out r2))
- {
- try
- {
+ if (Double.TryParse((string)vals[i], out r2)) {
+ try {
vals[i] = (Int16)r2;
continue;
- }
- catch { }
+ } catch { }
}
- }
- else if (t.Equals(typeof(Int32)))
- {
+ } else if (t.Equals(typeof(Int32))) {
Int32 result;
- if (Int32.TryParse((string)vals[i], out result))
- {
+ if (Int32.TryParse((string)vals[i], out result)) {
vals[i] = result;
continue;
}
double r2;
- if (Double.TryParse((string)vals[i], out r2))
- {
- try
- {
+ if (Double.TryParse((string)vals[i], out r2)) {
+ try {
vals[i] = (Int32)r2;
continue;
- }
- catch { }
+ } catch { }
}
- }
- else if (t.Equals(typeof(Int64)))
- {
+ } else if (t.Equals(typeof(Int64))) {
Int64 result;
- if (Int64.TryParse((string)vals[i], out result))
- {
+ if (Int64.TryParse((string)vals[i], out result)) {
vals[i] = result;
continue;
}
double r2;
- if (Double.TryParse((string)vals[i], out r2))
- {
- try
- {
+ if (Double.TryParse((string)vals[i], out r2)) {
+ try {
vals[i] = (Int64)r2;
continue;
- }
- catch { }
+ } catch { }
}
}
vals[i] = DBNull.Value;
@@ -247,8 +212,7 @@ namespace BCG
return new BlueRow(mySchema, vals);
}
- private Type myGetType(string type)
- {
+ private Type myGetType(string type) {
type = type.ToLower();
if (type.Contains("int"))
return typeof(int);
diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs
index d967c1409..ef9f2499f 100644
--- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs
+++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs
@@ -16,11 +16,73 @@ using BCG.Extensions;
using System.Text.Json;
using System.Text.Json.Nodes;
using Newtonsoft.Json;
+using BCG;
+using System.Runtime.CompilerServices;
+namespace bts {
+
+ // public class CompositeRow : IBlueRow {
+ // private IBlueRow _lrow, _rrow;
+ // private BlueRow _out_row;
+ // public CompositeRow(IBlueRow lrow, IBlueRow rrow) {
+ // _lrow = lrow;
+ // _rrow = rrow;
+ // IBlueTableSchema schema = _lrow.schema;
+ // List