diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddConstantColumnRow.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddConstantColumnRow.cs new file mode 100644 index 000000000..339919e68 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddConstantColumnRow.cs @@ -0,0 +1,12 @@ +namespace bts { + using BCG; + + using System; + using System.Collections; + + public class AddConstantColumnRow { + AddConstantColumnRow(IBlueRow r, Tuple[] constant_cols) { + + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs b/src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs index 33992859d..b87b3a5e9 100644 --- a/src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs @@ -11,7 +11,7 @@ using System.Reflection.Metadata; namespace BCG { public class SnapshotStream : ViewStream { - public enum SpeciicSnapshot { + public enum SpecificSnapshot { FirstOfDay, LastOfDay }; @@ -20,10 +20,19 @@ namespace BCG { public DateTime _snapshot_time; public string _snapshot_file; public IEnumerable _packets; - - public SnapshotStream(string specific_snapshot, string directory_name, string token) { - + public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) { + DateTime dt; + var parsed = DateTime.TryParse(snapshot_date, out dt); + dt = dt.Date; + bts.Log.assert(parsed); + SpecificSnapshot specific_snapshot_enum; + parsed = Enum.TryParse(specific_snapshot, out specific_snapshot_enum); + bts.Log.assert(parsed); + string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum); + ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token); + mySetRealSource(rs); } + public SnapshotStream(string snapshot_dt, string directory_name, string token) { _snapshot_time = PersistorStream.parse_datetime(snapshot_dt); string token_directory_name = $"{directory_name}\\{token}"; diff --git a/src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs b/src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs index 29631f591..31f1bf186 100644 --- a/src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs +++ b/src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs @@ -36,11 +36,21 @@ namespace BCG { } public IEnumerable packets = new List(); } - public static get_time(SnapshotStream.SpeciicSnapshot speciic_snapshot, string dir_name, string token) { - // TODO - return DateTime.MinValue; - } public static class StreamPersistorUtils { + public static string find_snapshot_file(string directory_name, DateTime snapshot_date, SnapshotStream.SpecificSnapshot specific_snapshot) { + DateTime max_dt = DateTime.MinValue; + string rv = String.Empty; + string[] file_names = Directory.GetFiles(directory_name); + for (int i = 0; i < file_names.Count() - 1; i++) { + string fn = file_names[i]; + var dt = parse_datetime_from_filename(fn); + if (dt > max_dt) { + max_dt = dt; + rv = fn; + } + } + return rv; + } public static string find_snapshot_file(string directory_name, DateTime snapshot_time) { string[] file_names = Directory.GetFiles(directory_name); for (int i = 0; i < file_names.Count() - 1; i++) { diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs index 0ae40b979..9b28c4c04 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs @@ -1,47 +1,42 @@ -// using System; -// using System.Collections; -// using BCG.Utilities; +using System; +using System.Collections; -// namespace BCG -// { -// /// -// /// Computes the max for a given column -// /// -// public class MaxGroupByEvaluator : GroupByEvaluator -// { +using BCG.Utilities; -// public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols) -// {} +namespace BCG { + /// + /// Computes the max for a given column + /// + public class MaxGroupByEvaluator : GroupByEvaluator { -// public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) -// : base (colMapEpression, outputCol, groupByCols) {} + public MaxGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { } -// public override IBlueRow evaluate (DataPacket dp) -// { -// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : -// myOutputTable; -// string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols); -// IBlueRow currMaxRow = myOutputTable.getRow (hashString); -// IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn); -// if (maxRow != null) -// { -// BlueEventLog.logAssert (currMaxRow != null); -// BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0); -// if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0) -// { -// maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn); -// if (maxRow == null) -// myOutputTable.removeRow (hashString); -// else -// myOutputTable.addRow (maxRow, hashString); -// } -// } -// maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn); -// // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true, -// // otherwise if maxRow > currMaxRow then its replaced -// if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0)) -// myOutputTable.addRow (maxRow, hashString); -// return myProject (myOutputTable.getRow (hashString)); -// } -// } -// } + public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) + : base(colMapEpression, outputCol, groupByCols) { } + + public override IBlueRow evaluate(DataPacket dp) { + myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : + myOutputTable; + string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols); + IBlueRow currMaxRow = myOutputTable.getRow(hashString); + IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn); + if (maxRow != null) { + bts.Log.assert(currMaxRow != null); + bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0); + if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) { + maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn); + if (maxRow == null) + myOutputTable.removeRow(hashString); + else + myOutputTable.addRow(maxRow, hashString); + } + } + maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn); + // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true, + // otherwise if maxRow > currMaxRow then its replaced + if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0)) + myOutputTable.addRow(maxRow, hashString); + return myProject(myOutputTable.getRow(hashString)); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs index 5ab927f41..9f3b44fc6 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs @@ -1,20 +1,15 @@ -// using System; +using System; -// namespace BCG -// { -// public class MaxStream : ViewStream -// { -// public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) -// { -// } -// public MaxStream (ViewStream stream, string outputCol, string[] inputCols) -// { -// mySetRealSource (new GroupByStream (stream, -// new MaxGroupByEvaluator (outputCol, inputCols))); -// } +namespace BCG { + public class MaxStream : ViewStream { + public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) { + } + public MaxStream(ViewStream stream, string outputCol, string[] inputCols) { + mySetRealSource(new GroupByStream(stream, + new MaxGroupByEvaluator(outputCol, inputCols))); + } -// public MaxStream (ViewStream stream, string outputCol) -// : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"}) -// {} -// } -// } + // public MaxStream(ViewStream stream, string outputCol) + // : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs index dc6700e22..541e79d67 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs @@ -1,40 +1,36 @@ -// using System; -// using BCG.Utilities; +using System; -// namespace BCG -// { -// public class MinGroupByEvaluator : MaxGroupByEvaluator -// { -// public MinGroupByEvaluator (string outputCol, string[] groupByCols) -// : base (outputCol, groupByCols) {} +using BCG.Utilities; -// public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) : -// base (colMapEpression, outputCol, groupByCols) {} +namespace BCG { + public class MinGroupByEvaluator : MaxGroupByEvaluator { + public MinGroupByEvaluator(string outputCol, string[] groupByCols) + : base(outputCol, groupByCols) { } -// public override IBlueRow evaluate (DataPacket dp) -// { -// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable; -// string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols); -// IBlueRow currMinRow = myOutputTable.getRow (stringRep); -// IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn); -// if (minRow != null) -// { -// // BlueEventLog.logAssert (currMinRow != null); -// // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0); -// if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0) -// { -// minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn); -// if (minRow == null) -// myOutputTable.removeRow (stringRep); -// else -// myOutputTable.addRow (minRow, stringRep); -// } -// } -// minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn); -// if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0)) -// myOutputTable.addRow (minRow, stringRep); -// return myOutputTable.getRow (stringRep); -// } + public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) : + base(colMapEpression, outputCol, groupByCols) { } -// } -// } + public override IBlueRow evaluate(DataPacket dp) { + myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable; + string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols); + IBlueRow currMinRow = myOutputTable.getRow(stringRep); + IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn); + if (minRow != null) { + // BlueEventLog.logAssert (currMinRow != null); + // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0); + if (BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) == 0) { + minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn); + if (minRow == null) + myOutputTable.removeRow(stringRep); + else + myOutputTable.addRow(minRow, stringRep); + } + } + minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn); + if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) < 0)) + myOutputTable.addRow(minRow, stringRep); + return myOutputTable.getRow(stringRep); + } + + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs index 674c5b0de..bb3d47495 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs @@ -1,17 +1,13 @@ -// using System; +using System; -// namespace BCG -// { -// public class MinStream : ViewStream -// { -// public MinStream (ViewStream stream, string outputCol, string[] inputCol) -// { -// mySetRealSource (new GroupByStream (stream, -// new MinGroupByEvaluator (outputCol, inputCol))); -// } +namespace BCG { + public class MinStream : ViewStream { + public MinStream(ViewStream stream, string outputCol, string[] inputCol) { + mySetRealSource(new GroupByStream(stream, + new MinGroupByEvaluator(outputCol, inputCol))); + } -// public MinStream (ViewStream stream, string outputCol) -// : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"}) -// {} -// } -// } + // public MinStream(ViewStream stream, string outputCol) + // : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs index 562e04799..d791dea7b 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs @@ -1,94 +1,82 @@ -// using System; +using System; -// namespace BCG -// { -// /// -// /// Computes the sum for a given column -// /// -// public class SumGroupByEvaluator : GroupByEvaluator -// { -// public SumGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols) -// {} +namespace BCG { + /// + /// Computes the sum for a given column + /// + public class SumGroupByEvaluator : GroupByEvaluator { + public SumGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { } -// public SumGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) -// : base (colMapEpression, outputCol, groupByCols) {} + public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) + : base(colMapEpression, outputCol, groupByCols) { } -// public override IBlueRow evaluate (DataPacket dp) -// { -// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable; - -// IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn); -// return sumRow; -// } -// } + public override IBlueRow evaluate(DataPacket dp) { + myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable; + + IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn); + return sumRow; + } + } -// public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator -// { -// private string[] myOutputCols, myGroupByCols; -// public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) -// { -// myOutputCols = spaceSeperatedOutputCols.Split(' '); -// myGroupByCols = spaceSeperatedGroupByCols.Split(' '); -// } + public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator { + private string[] myOutputCols, myGroupByCols; + public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) { + myOutputCols = spaceSeperatedOutputCols.Split(' '); + myGroupByCols = spaceSeperatedGroupByCols.Split(' '); + } -// public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) -// { -// myOutputCols = output_columns; -// myGroupByCols = groupby_columns; -// } + public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) { + myOutputCols = output_columns; + myGroupByCols = groupby_columns; + } -// #region IGroupByEvaluator Members + #region IGroupByEvaluator Members -// public IBlueRow evaluate(DataPacket packet) {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);} + public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); } -// public string[] groupByColumns { get { return myGroupByCols; } } + public string[] groupByColumns { get { return myGroupByCols; } } -// public void clear() {/*stateless*/} + public void clear() {/*stateless*/} -// public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) {return schema;} + public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; } -// #endregion -// } + #endregion + } -// public class SumGroupByEvaluator : GroupByEvaluator -// { -// public SumGroupByEvaluator(string outputCol, string[] groupByCols) -// : base(outputCol, groupByCols) -// { } + public class SumGroupByEvaluator : GroupByEvaluator { + public SumGroupByEvaluator(string outputCol, string[] groupByCols) + : base(outputCol, groupByCols) { } -// public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) -// : base(colMapEpression, outputCol, groupByCols) { } + public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) + : base(colMapEpression, outputCol, groupByCols) { } -// public override IBlueRow evaluate(DataPacket dp) -// { -// myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable; + public override IBlueRow evaluate(DataPacket dp) { + myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable; -// IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn); -// return sumRow; -// } -// } + IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn); + return sumRow; + } + } -// public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator -// { -// private string[] myOutputCols, myGroupByCols; -// public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) -// { -// myOutputCols = spaceSeperatedOutputCols.Split(' '); -// myGroupByCols = spaceSeperatedGroupByCols.Split(' '); -// } + public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator { + private string[] myOutputCols, myGroupByCols; + public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) { + myOutputCols = spaceSeperatedOutputCols.Split(' '); + myGroupByCols = spaceSeperatedGroupByCols.Split(' '); + } -// #region IGroupByEvaluator Members + #region IGroupByEvaluator Members -// public IBlueRow evaluate(DataPacket packet) {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);} + public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); } -// public string[] groupByColumns { get { return myGroupByCols; } } + public string[] groupByColumns { get { return myGroupByCols; } } -// public void clear() {/*stateless*/} + public void clear() {/*stateless*/} -// public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) {return schema;} + public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; } -// #endregion -// } + #endregion + } -// } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs index 238cbd27a..dced8d6f5 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs @@ -1,107 +1,89 @@ -// using System; -// using System.Collections.Generic; +using System; +using System.Collections.Generic; -// namespace BCG -// { -// public class NullMapper : IStreamMapper -// { -// private Dictionary _valid_cols = new Dictionary(); +namespace BCG { + public class NullMapper : IStreamMapper { + private Dictionary _valid_cols = new Dictionary(); -// public NullMapper(string[] valid_cols) -// { -// foreach (string c in valid_cols) -// _valid_cols[c] = true; -// _valid_cols["id"] = true; -// } + public NullMapper(string[] valid_cols) { + foreach (string c in valid_cols) + _valid_cols[c] = true; + _valid_cols["id"] = true; + } -// public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; } + public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; } -// public IBlueRow map(IBlueRow row) -// { -// MutableBlueRow out_row = new MutableBlueRow(row); -// foreach (string c in row.schema.columnNames) -// if (!_valid_cols.ContainsKey(c)) -// out_row.setValue(c, DBNull.Value); -// return out_row; -// } -// } + public IBlueRow map(IBlueRow row) { + MutableBlueRow out_row = new MutableBlueRow(row); + foreach (string c in row.schema.columnNames) + if (!_valid_cols.ContainsKey(c)) + out_row.setValue(c, DBNull.Value); + return out_row; + } + } -// public class SumStream : ViewStream -// { -// public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) -// { -// string[] outputCols = spaceSeparatedOutputCols.Split(' '); -// string[] groupByCols = spaceSeparatedGroupByCols.Split(' '); -// mySetRealSource(new GroupByStream(stream, -// outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator : -// new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols))); -// } + public class SumStream : ViewStream { + public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) { + string[] outputCols = spaceSeparatedOutputCols.Split(' '); + string[] groupByCols = spaceSeparatedGroupByCols.Split(' '); + mySetRealSource(new GroupByStream(stream, + outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator : + new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols))); + } -// public SumStream(ViewStream stream, string[] output_columns) -// { -// ViewStream rs = new AddColumnsStream(stream, "fake_col=0"); -// rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false); -// rs = new ProjectStream(rs, "!fake_col"); -// mySetRealSource(rs); -// } + // public SumStream(ViewStream stream, string[] output_columns) { + // ViewStream rs = new AddColumnsStream(stream, "fake_col=0"); + // rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false); + // rs = new ProjectStream(rs, "!fake_col"); + // mySetRealSource(rs); + // } -// public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) -// { -// string[] output_columns = space_seperated_output_columns.Split(' '); -// string[] groupby_columns = String.IsNullOrEmpty (space_seperated_groupby_columns) ? new string[]{} : space_seperated_groupby_columns.Split (' '); -// mySetRealSource (new SumStream (stream, output_columns, groupby_columns, force_null_into_other_columns)); -// } + public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) { + string[] output_columns = space_seperated_output_columns.Split(' '); + string[] groupby_columns = String.IsNullOrEmpty(space_seperated_groupby_columns) ? new string[] { } : space_seperated_groupby_columns.Split(' '); + mySetRealSource(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns)); + } -// public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) -// { -// ViewStream rs = null; -// groupby_columns = groupby_columns ?? new string[] {}; -// if (groupby_columns.Length == 0) -// { -// rs = new AddColumnsStream(stream, "fake_col=0"); -// rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] {"fake_col"})); -// rs = new ProjectStream(rs, "!fake_col"); -// } -// else -// { -// rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns)); -// } -// if (force_null_into_other_columns) -// { -// List valid_cols = new List(output_columns); -// valid_cols.AddRange(groupby_columns); -// rs = new MapStream(rs, new NullMapper(valid_cols.ToArray())); -// } -// mySetRealSource(rs); -// } + public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) { + ViewStream rs = null; + groupby_columns = groupby_columns ?? new string[] { }; + if (groupby_columns.Length == 0) { + // rs = new AddColumnsStream(stream, "fake_col=0"); + // rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" })); + // rs = new ProjectStream(rs, "!fake_col"); + } else { + rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns)); + } + if (force_null_into_other_columns) { + List valid_cols = new List(output_columns); + valid_cols.AddRange(groupby_columns); + rs = new MapStream(rs, new NullMapper(valid_cols.ToArray())); + } + mySetRealSource(rs); + } -// public SumStream (ViewStream stream, string outputCol) -// : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, "fakeCol") -// { -// } -// } + // public SumStream(ViewStream stream, string outputCol) + // : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") { + // } + } -// public class IntSumStream : ViewStream -// { -// public IntSumStream (ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) -// { -// string[] outputCols = spaceSeparatedOutputCols.Split(' '); -// string[] groupByCols = spaceSeparatedGroupByCols.Split(' '); -// mySetRealSource(new GroupByStream(stream, -// outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator: -// new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols))); -// } + public class IntSumStream : ViewStream { + public IntSumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) { + string[] outputCols = spaceSeparatedOutputCols.Split(' '); + string[] groupByCols = spaceSeparatedGroupByCols.Split(' '); + mySetRealSource(new GroupByStream(stream, + outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator : + new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols))); + } -// } + } -// public class SumStream : ViewStream -// { -// public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) -// { -// string[] inputCols = spaceSeparatedInputCols.Split(' '); -// mySetRealSource(new GroupByStream(stream, -// new SumGroupByEvaluator(outputCol, inputCols))); -// } -// } -// } + public class SumStream : ViewStream { + public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) { + string[] inputCols = spaceSeparatedInputCols.Split(' '); + mySetRealSource(new GroupByStream(stream, + new SumGroupByEvaluator(outputCol, inputCols))); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/InterRow/RowDiffStream.cs b/src/DataStreamSuite/DataStream/Primitives/InterRow/RowDiffStream.cs new file mode 100644 index 000000000..9687a0c0d --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/InterRow/RowDiffStream.cs @@ -0,0 +1,79 @@ +using BCG; + +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +using ZstdSharp.Unsafe; + +namespace bts { + public class RowDiffStream : ViewStream { + private ViewStream _vs; + private readonly string _order_col, _col_1, _col_2; + private readonly int _diff; + private IBlueTableSchema _out_schema; + private readonly string _diff_col_name; + public RowDiffStream(ViewStream vs, string order_col, string col_1, string col_2, int diff, string diff_col_name) { + _vs = vs; + _order_col = order_col; + _col_1 = col_1; + _col_2 = col_2; + _diff = diff; + _diff_col_name = diff_col_name; + } + + private string _diff_col_name_perc { + get { + return $"{_diff_col_name}_percentage"; + } + } + + protected override void myOnFirstSubscriber() { + _vs.onChange += _handle_vs; + } + + protected override void myOnLastSubscriber() { + _vs.onChange -= _handle_vs; + } + + private void _handle_vs(object dp_obj) { + var dp = dp_obj as DataPacket; + IBlueRowCollection nrows = dp.newRows; + _out_schema = BlueDataUtility.add_column(nrows.schema, _diff_col_name, typeof(double)); + _out_schema = BlueDataUtility.add_column(_out_schema, _diff_col_name_perc, typeof(double)); + List row_list = new List(); + foreach (IBlueRow r in nrows) + row_list.Add(r); + //OrderBy sorts in ascedning order + var sorted_rows = row_list.OrderBy(r => r[_order_col]).ToList(); + var sorted_row_array = sorted_rows.ToArray(); + // reverse it to make it descending + sorted_rows.Reverse(); + //sort rows so we can do the diff + OrderedBlueTable rv = new OrderedBlueTable(_out_schema); + int i = 0; + foreach (IBlueRow r in sorted_rows) { + MutableBlueRow nr = new MutableBlueRow(_out_schema, r); + //So, in the expression "x - y," where y is subtracted from x, x is the minuend, and y is the subtrahend. + // we assume x - y, and name variables like that + double x = r.get(_col_1); + int y_row_idx = i + _diff; + double diff = System.Double.NaN; + double perc = System.Double.NaN; + if (y_row_idx < sorted_rows.Count()) { + var y = sorted_rows[y_row_idx].get(_col_2); + diff = x - y; + perc = 100.0 * diff / y; + } + nr.setValue(_diff_col_name, diff); + nr.setValue(_diff_col_name_perc, perc); + rv.addRow(nr); + i++; + } + + var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true); + myFireEvent(dp_out); + + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs new file mode 100644 index 000000000..c92fcf72b --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs @@ -0,0 +1,34 @@ +using BCG; + +namespace bts { + public class SchemaGuesserStream : ViewStream { + + private IBlueRowCollection _rows = null; + private ViewStream _vs; + public SchemaGuesserStream(ViewStream vs) { + _vs = vs; + } + + protected override void myOnFirstSubscriber() { + _vs.onChange += _on_data; + } + protected override void myOnLastSubscriber() { + _vs.onChange -= _on_data; + } + + private void _on_data(object obj) { + DataPacket dp = obj as DataPacket; + if (_rows == null) { + _rows = BlueDataUtility.get_typed_table(dp.newRows); + } + var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket); + myFireEvent(dp_out); + + + } + + + + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs b/src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs index eee047756..b45fb0b29 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs @@ -61,7 +61,7 @@ // { // syntax parser = // new syntax(); - + // mySortCols = ((SortExpression) // parser.Parse (sortExpression)).columns; // } @@ -127,3 +127,61 @@ // } // } // } + +using BCG; + +using System.Collections; +using System.Collections.Generic; +using System.Linq; + + +[DataStreamUserRespresentation(friendlyName: "sort")] +public class SortStreamTemp : ViewStream { + private ViewStream _vs; + private readonly string _sort_col_name; + private readonly bool _is_asc = true; + public SortStreamTemp(ViewStream vs, string sort_exp) { + _vs = vs; + if (sort_exp.StartsWith("+")) { + _is_asc = true; + _sort_col_name = sort_exp.Substring(1); + } else if (sort_exp.StartsWith("-")) { + _is_asc = false; + _sort_col_name = sort_exp.Substring(1); + } else { + _is_asc = true; + _sort_col_name = sort_exp; + } + } + + protected override void myOnFirstSubscriber() { + _vs.onChange += _handle_vs_update; + } + + protected override void myOnLastSubscriber() { + _vs.onChange -= _handle_vs_update; + } + + private void _handle_vs_update(object dp_obj) { + var dp = dp_obj as DataPacket; + IBlueRowCollection nrows = dp.newRows; + var _out_schema = nrows.schema; + List row_list = new List(); + foreach (IBlueRow r in nrows) + row_list.Add(r); + //OrderBy sorts in ascedning order + var sorted_rows = row_list.OrderBy(r => r[_sort_col_name]).ToList(); + if (!_is_asc) + // reverse it to make it descending + sorted_rows.Reverse(); + //sort rows so we can do the diff + OrderedBlueTable rv = new OrderedBlueTable(_out_schema); + foreach (IBlueRow r in sorted_rows) { + rv.addRow(r); + } + + var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true); + myFireEvent(dp_out); + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs index 1a4762ffa..4972db6c0 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs @@ -6,6 +6,7 @@ using System.Reflection; using System.IO; using System.Diagnostics; using System.Linq; +using System.Globalization; //using BCG.Utilities; using BCG.Utilities.General; @@ -292,21 +293,118 @@ namespace BCG { return TypeCode.Empty; } - public static IBlueTableSchema to_typed_schema(IBlueRowCollection rows) { - List cis = new List(); - foreach (var col in rows.schema.columnInfo) { - var text_vals = BlueDataUtility.get_colvalues(rows, col.colName); - TypeCode type_code = convert_column_type(text_vals); - cis.Add(new ColumnInfo(col.colName, cis.Count, _typecode_to_type[type_code])); + // public static Type guess_type(string val) { + // //DateTime + // { + // DateTime dt; + // var success = DateTime.TryParseExact(val, "yyyy-MM-dd HH:mm:ss", null, DateTimeStyles.AllowWhiteSpaces, out dt); + // if (success) { + // return typeof(DateTime); + // } + // DateTimeOffset dtoff; + // success = DateTimeOffset.TryParseExact(val, "yyyy-MM-dd HH:mm:sszzz", + // null, System.Globalization.DateTimeStyles.None, out dtoff); + + // if (success) { + // return typeof(DateTime); + // } + // } + // //int + // { + // int i; + // var success = Int32.TryParse(val, out i); + // if (success) { + // return typeof(Int32); + // } + // } + // //double + // { + // double d; + // var success = Double.TryParse(val, out d); + // if (success) { + // return typeof(Double); + // } + // } + // //bool + // { + // bool b; + // var success = Boolean.TryParse(val, out b); + // if (success) { + // return typeof(bool); + // } + // } + // //string + // return typeof(string); + // } + + private static Type[] _parsable_types = { typeof(DateTime), typeof(int), typeof(double), typeof(bool) }; + public static Type guess_type(string val) { + foreach (var t in _parsable_types) { + var type_val = convert_to_type(t, val); + if (type_val != null) + return t; } - return new BlueTableSchema(cis.ToArray(), rows.schema.tableName); + return typeof(string); + } - // public static IBlueRow convert_string_row(IBlueRow r, IBlueTableSchema out_schema) { - // foreach (var ci in out_schema.columnInfo){ + public static IBlueTableSchema get_typed_schema(IBlueRowCollection rows) { + // just use one row to guess + List cis = new List(); + foreach (IBlueRow r in rows) { + foreach (var ci in r.schema.columnInfo) { + if (ci.colType != typeof(System.String)) { + cis.Add(ci); + continue; + } + Type type = guess_type(r[ci.colName] as string); + cis.Add(new ColumnInfo(ci.colName, ci.colNo, type)); + } + return new BlueTableSchema(cis.ToArray(), rows.schema.tableName); + } + return null; + } + + static object convert_to_type(Type t, string value) { + // Attempt to convert the string to the specified type + try { + return Convert.ChangeType(value, t); + } catch (Exception) { + if (t == typeof(double)) { + // some doubles come with currency and ',' we clean that up here (there might be better ways to do this) + string cleaned_val = new string(value.Where(c => c != '$' && c != ',').ToArray()); + cleaned_val = cleaned_val.Replace('−', '-'); + double rv_d; + if (Double.TryParse(cleaned_val, out rv_d)) + return rv_d; + else + return null; + } + return null; + // Handle conversion failure gracefully + // throw new ArgumentException("Failed to convert the string to the specified type."); + } + } + + // converts a row where all columns are string types to a schema where the best types are guesed + public static IBlueRowCollection get_typed_table(IBlueRowCollection string_rows) { + Func get_typed_row = (IBlueRow string_row, IBlueTableSchema typed_schema) => { + MutableBlueRow rv = new MutableBlueRow(typed_schema); + foreach (ColumnInfo ci in typed_schema.columnInfo) { + var typed_val = convert_to_type(ci.colType, value: string_row[ci.colName].ToString()); + rv.setValue(ci.colName, typed_val); + } + return rv; + }; + IBlueTableSchema typed_schema = get_typed_schema(string_rows); + var rv = new BlueTable(typed_schema); + foreach (IBlueRow r in string_rows) { + var typed_row = get_typed_row(r, typed_schema); + rv.addRow(typed_row); + } + return rv; + } - // } - // } /// @@ -2112,5 +2210,33 @@ namespace BCG { } #endregion + + public class RowComparer : IComparer { + private readonly IEnumerable> _sort_cols; + public RowComparer(IEnumerable> sort_cols) { + _sort_cols = sort_cols; + } + public int Compare(IBlueRow? x, IBlueRow? y) { + Func compare_cols = (o1, o2) => { + return (o1 as IComparable).CompareTo(o2); + }; + foreach (var c in _sort_cols) { + int i = compare_cols(x[c.Item1], y[c.Item1]); + + } + return 1; + } + + } + public static IList get_sorted_row(IBlueRowCollection rows, IEnumerable> sort_cols) { + List row_list = new List(); + foreach (IBlueRow r in rows) { + row_list.Add(r); + } + var row_comparer = new RowComparer(sort_cols); + ParallelSort.QuicksortSequential(row_list, row_comparer); + return row_list; + + } } } diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs index c8acd60d4..f0eb52994 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Program.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs @@ -6,7 +6,10 @@ using Bts.DataStreamExpressions; using System.Collections.Generic; using BCG; -using System.Threading; +using System.IO; +using BCG.Utilities.General; +using bts; +using System.Linq; bts.Log.Gen.info($"Starting DSPublisher"); @@ -57,6 +60,8 @@ static void start_echo_client() { void _publish_streams() { PubTableBroker pb_broker = new PubTableBroker(); CloudStreamManager csm = new CloudStreamManager(); + string csv_folder = @"C:\Temp\csvs_topublish"; + CSVPublisher csv_publisher = new CSVPublisher(csv_folder, pb_broker); // get it from config Dictionary streams_to_publish = new Dictionary() { //{"bbo","bbo()"}, @@ -88,4 +93,37 @@ void _publish_streams() { void on_change(object obj) { bts.Log.Gen.info($"DSPublisher on_change dp: {obj}"); +} + +public class CSVPublisher : IWorker { + private IDictionary _pubs = new Dictionary(); + // read from config + private readonly string _csv_directory = @"C:\Temp\csvs_topublish\"; + + private PubTableBroker _pub_table_broker; + public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) { + _csv_directory = csv_directory; + _pub_table_broker = pub_table_broker; + MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); + } + public void doWork() { + var file_names = Directory.GetFiles(_csv_directory); + foreach (var fn in file_names) { + if (!_pubs.ContainsKey(fn)) { + _handle_new_file(fn); + } + } + } + + private void _handle_new_file(string file_name) { + if (file_name.EndsWith(".csv")) { + ViewStream vs = new CSVReaderStream(file_name); + vs = new SchemaGuesserStream(vs); + var toks = file_name.Split('\\'); + string table_name = toks.Last(); + table_name = table_name.Remove(table_name.Length - ".csv".Length); + ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs); + _pubs.Add(file_name, vp); + } + } } \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/TradeBarsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/TradeBarsStream.cs index 5e34457c0..5fa0cf634 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/TradeBarsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/TradeBarsStream.cs @@ -28,7 +28,7 @@ namespace bts { } public void add_trade(IBlueRow r) { double price = r.get("price"); - prices.Add(price); + //prices.Add(price); double qty = r.get(col_name: "qty"); if (n_entries == 0) { open = price; @@ -55,8 +55,8 @@ namespace bts { } public double volatility { get { - if (prices.Count == 1) - return 0.0; + //if (prices.Count == 1) + return 0.0; var log_returns = calculate_log_returns(prices.ToArray()); return calculate_volatility(log_returns); } diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs index cae7a7f11..80d9742da 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs @@ -43,11 +43,13 @@ namespace BCG { private IDisposable _sub = null; private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo", Tuple.Create("id", TypeCode.Int32), - Tuple.Create("sec_key", TypeCode.Int32), + Tuple.Create("sec_key", TypeCode.UInt32), Tuple.Create("sec", TypeCode.String), - Tuple.Create("open", TypeCode.UInt32), + Tuple.Create("date", TypeCode.DateTime), + Tuple.Create("open", TypeCode.Double), Tuple.Create("close", TypeCode.Double), - Tuple.Create("settlement", TypeCode.Single)); + Tuple.Create("settlement", TypeCode.Double)); + IDisposable? subscription; // TODO: add cons for symbol, exchange etc public PriceStatsStream() @@ -62,7 +64,7 @@ namespace BCG { protected override void myOnFirstSubscriber() { _vsd.addWorker(this); - _sub = _md_client.price_stats().Subscribe(_on_price_stats); + subscription = _md_client.price_stats().Subscribe(_on_price_stats); } private void _log_updates(TableUpdate update) { @@ -81,7 +83,7 @@ namespace BCG { // just cacching in here, will populate the rows in _poll_scan and fire the datapacket bts.Log.Gen.info($"PriceStatsStream._on_price_stats state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}"); _cached_state = update.state; - _dirty = false; + _dirty = true; } protected override void myOnLastSubscriber() { @@ -99,7 +101,8 @@ namespace BCG { SecurityKey skey = kv.Key; int id = Convert.ToInt32(skey); PriceStats pstats = kv.Value; - IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), pstats.open, pstats.close, pstats.settlement); + var today = DateTime.Now.Date; + IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), today, pstats.open, pstats.close, pstats.settlement); new_state.addRow(r); } DataPacket dp = null; @@ -124,4 +127,4 @@ namespace BCG { } } } -} \ No newline at end of file +}