diff --git a/src/DataStreamSuite/DataStream/Offloading/AddToCloudStream.cs b/src/DataStreamSuite/DataStream/Offloading/AddToCloudStream.cs deleted file mode 100644 index 0455d40cc..000000000 --- a/src/DataStreamSuite/DataStream/Offloading/AddToCloudStream.cs +++ /dev/null @@ -1,32 +0,0 @@ -using BCG; - -namespace BCG -{ - // this is a stream so we can run it from the DSQA (since dsqa can only load streams) - [DataStreamUserRespresentation("add_to_cloud")] - public class AddToCloudStream : ViewStream - { - private readonly string _name, _stream_exp; - private StreamOffloadRequestor _requestor = new StreamOffloadRequestor(); - public AddToCloudStream(string name, params string[] lines) - : this(name, 0, lines) - { - } - - public AddToCloudStream(string name, int persist, params string[] lines) - { - _name = name; - _stream_exp = getExpression(lines); - _requestor.add_to_cloud(_name, _stream_exp, persist == 1); - mySetRealSource(new TableStream("cloud_streams")); - } - - static string getExpression(params string[] lines) { - for (int i = 0; i < lines.Length; i++) { - // unescape quotes - lines[i] = lines[i].Replace("\\\"", "\""); - } - return string.Join("\n", lines); - } - } -} diff --git a/src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs b/src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs index ae4bfca6a..effb56f1d 100644 --- a/src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs +++ b/src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs @@ -3,23 +3,20 @@ using System.Collections.Generic; using Bts.DataStreamExpressions; using BCG.Utilities.General; +using bts; -namespace BCG -{ +namespace BCG { // listens to the requests posted on "cloud_streams" request table, and // if unfulfilled starts a ViewPubsliher - public class CloudStreamManager : IWorker - { - public class StreamPubUnit - { + public class RemoteQueryManager : IWorker { + public class StreamPubUnit { public string name; public string stream_expression; public ViewStream stream; public bool persist; public ViewPublisher view_publisher; - public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) - { + public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) { name = name_; stream_expression = stream_expression_; stream = stream_; @@ -35,9 +32,7 @@ namespace BCG var rv = exp.getView(new Dictionary(0)); return rv; - } - catch (Exception ex) - { + } catch (Exception ex) { bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}"); return null; } @@ -46,11 +41,15 @@ namespace BCG _stream_pub_map = new Dictionary(); private ViewStream _stream = new TableStream("cloud_streams"); private PubTableBroker _broker = new PubTableBroker(); + private ManagedSubscriber _sub = ManagedSubscriber.instance; - public CloudStreamManager() - { + + public RemoteQueryManager() { _stream.onChange += _on_str; - + _sub.subscribe("RemoteRequest", (string tag, byte[] data) => { + RemoteRequest req = RemoteRequest.construct_from_bytes(data); + _spawn_remote_stream(req.published_table_name, req.stream_expression, false); + }); } private void _on_str(object obj) { @@ -62,20 +61,17 @@ namespace BCG string name = r.get("name"); bool persist = r.get("persist"); string stream_exp = r.get("stream_exp"); - if (!_stream_pub_map.ContainsKey(name)) - { - _assign(name, stream_exp, persist); + if (!_stream_pub_map.ContainsKey(name)) { + _spawn_remote_stream(name, stream_exp, persist); } } } - private void _assign(string name, string stream_expression, bool persist) - { + private void _spawn_remote_stream(string name, string stream_expression, bool persist) { bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}"); ViewStream stream = build_stream(stream_expression); _expose_to_ws(name, stream); - if (persist) - { + if (persist) { stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs } ViewPublisher view_pub = new ViewPublisher(name, _broker, stream); @@ -83,13 +79,11 @@ namespace BCG _stream_pub_map.Add(name, u); } - private void _expose_to_ws(string name, ViewStream stream) - { + private void _expose_to_ws(string name, ViewStream stream) { // TODO: john } - public void doWork() - { + public void doWork() { } } diff --git a/src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs b/src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs new file mode 100644 index 000000000..47a3b9433 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Offloading/RemoteQueryStream.cs @@ -0,0 +1,48 @@ +using BCG; + +using bts; + +using System; + +namespace BCG { + // this is a stream so we can run it from the DSQA (since dsqa can only load streams) + [DataStreamUserRespresentation("remoteQuery")] + public class RemoteQueryStream : ViewStream { + private readonly string _name, _stream_exp; + private PublisherCore _publisher_core = new PublisherCore(); + private ViewStream _source_str; + + public RemoteQueryStream(params string[] lines) { + _stream_exp = get_expression(lines); + _name = Guid.NewGuid().ToString(); + _name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}"; + //_name.Replace("-", "_"); + + } + + protected override void myOnFirstSubscriber() { + RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name); + byte[] rem_req_bytes = rem_req.serialize(); + _publisher_core.publish("System", "RemoteRequest", rem_req_bytes); + _source_str = new TableStream(_name); + _source_str.onChange += new ChangeDelegate(_handle_data); + } + + protected override void myOnLastSubscriber() { + //maybe there should be a way to turn remote requests off ? + _source_str.onChange -= new ChangeDelegate(_handle_data); + } + + static string get_expression(params string[] lines) { + for (int i = 0; i < lines.Length; i++) { + // unescape quotes + lines[i] = lines[i].Replace("\\\"", "\""); + } + return string.Join("\n", lines); + } + + private void _handle_data(object o) { + myFireEvent(o); + } + } +} diff --git a/src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs b/src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs new file mode 100644 index 000000000..51e612d75 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs @@ -0,0 +1,31 @@ +namespace bts { + using System.IO; + using BCG; + + using System.IO; + using System.Threading.Tasks.Dataflow; + + public class RemoteRequest { + public readonly string stream_expression; + public readonly string published_table_name; + public RemoteRequest(string stream_expression_, string published_table_name_) { + stream_expression = stream_expression_; + published_table_name = published_table_name_; + } + public byte[] serialize() { + var str = new MemoryStream(); + var bw = new BinaryWriter(str); + bw.write_string_cpp(stream_expression); + bw.write_string_cpp(published_table_name); + return str.ToArray(); + } + + public static RemoteRequest construct_from_bytes(byte[] ser_bytes) { + BinaryReader br = new BinaryReader(new MemoryStream(ser_bytes)); + string stream_expression_ = br.read_string_cpp(); + string published_table_name_ = br.read_string_cpp(); + return new RemoteRequest(stream_expression_, published_table_name_); + + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs b/src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs index f6b220e41..5550d5bec 100644 --- a/src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs +++ b/src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs @@ -1,53 +1,54 @@ -using BCG; +// using BCG; -using bts; +// using bts; -using System.Linq; +// using System.Linq; -namespace BCG { - public class StreamOffloadRequestor { - public class OffloadOp { - string name; - string stream_exp; - } - private IBlueTableSchema _schema; - private IBlueRowCollection _rows; - private PubTableBroker _broker = new PubTableBroker(); - ViewStream _existing_offloads_vs = new TableStream("cloud_streams"); - public StreamOffloadRequestor() { - _schema = new BlueTableSchema("cloud_streams", - new ColumnInfo("id", 0, typeof(int)), - new ColumnInfo("name", 1, typeof(string)), - new ColumnInfo("persist", 2, typeof(bool)), - new ColumnInfo("stream_exp", 3, typeof(string))); - _existing_offloads_vs.onChange += _on_str; - } - private void _on_str(object obj) { - DataPacket dp = obj as DataPacket; - _rows = dp.newRows; - // this is needed so the broker knows the state of the table - _broker.create_table(_rows); - } - public void add_to_cloud(string name, string stream_exp, bool persist) { - // check if it exists - if (_rows == null) { - BlueTable table = new BlueTable(_schema); - IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp); - table.addRow(nr); - _broker.create_table(table); - // create table - } else { - if (exists(name)) - return; - IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp); - // TODO: use make_batch_changes instead - Log.assert(false, "Fix addRow"); - //_broker.addRow(nr); - } - } +// namespace BCG { +// // used on the client +// public class StreamOffloadRequestor { +// public class OffloadOp { +// string name; +// string stream_exp; +// } +// private IBlueTableSchema _schema; +// private IBlueRowCollection _rows; +// private PubTableBroker _broker = new PubTableBroker(); +// ViewStream _existing_offloads_vs = new TableStream("cloud_streams"); +// public StreamOffloadRequestor() { +// _schema = new BlueTableSchema("cloud_streams", +// new ColumnInfo("id", 0, typeof(int)), +// new ColumnInfo("name", 1, typeof(string)), +// new ColumnInfo("persist", 2, typeof(bool)), +// new ColumnInfo("stream_exp", 3, typeof(string))); +// _existing_offloads_vs.onChange += _on_str; +// } +// private void _on_str(object obj) { +// DataPacket dp = obj as DataPacket; +// _rows = dp.newRows; +// // this is needed so the broker knows the state of the table +// _broker.create_table(_rows); +// } +// public void add_to_cloud(string name, string stream_exp, bool persist) { +// // check if it exists +// if (_rows == null) { +// BlueTable table = new BlueTable(_schema); +// IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp); +// table.addRow(nr); +// _broker.create_table(table); +// // create table +// } else { +// if (exists(name)) +// return; +// IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp); +// // TODO: use make_batch_changes instead +// Log.assert(false, "Fix addRow"); +// //_broker.addRow(nr); +// } +// } - bool exists(string name) { - return _rows.to_ienum().Where(r => name == r.get("name")).Count() > 0; - } - } -} \ No newline at end of file +// bool exists(string name) { +// return _rows.to_ienum().Where(r => name == r.get("name")).Count() > 0; +// } +// } +// } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs index 3c79de595..321087674 100644 --- a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs @@ -1,6 +1,34 @@ -namespace BCG; - -public class AddColumnsStream : MapStream { - public AddColumnsStream(ViewStream vs, string columnDef) - : base(vs, new ColumnAdder(columnDef)) { } -} +using System; + +namespace BCG { + public class AddColumnsStream : ViewStream { + public static Tuple parse_exp(string exp) { + string[] toks = exp.Split(':'); + string col_name = toks[0]; + string col_value = toks[1]; + Tuple tup = BlueDataUtility.guess_type(col_value); + return Tuple.Create(col_name, tup.Item1, tup.Item2); + } + + private Tuple _new_col; + private IBlueTableSchema _out_schema; + + public AddColumnsStream(ViewStream vs, string col_def) { + _new_col = parse_exp(col_def); + ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r)); + mySetRealSource(rs); + } + + private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) { + if (_out_schema == null) + _out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2); + return _out_schema; + } + + private IBlueRow _map(IBlueRow r) { + MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r); + rv.setValue(_new_col.Item1, _new_col.Item3); + return rv; + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs b/src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs index bf7a0ea50..8cb3ebc1c 100644 --- a/src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs @@ -1,48 +1,24 @@ -using System; -using System.Collections.Generic; -using System.Text; -using System.IO; +using BCG; -using System.Runtime.Serialization.Formatters.Binary; - -namespace BCG -{ +namespace bts { [DataStreamUserRespresentation("loader")] - public class LoaderStream : ViewStream, ISideEffectStream - { - public class Exception : System.Exception - {public Exception(string msg) : base(msg) { }} + public class LoaderStream : ViewStream { - public LoaderStream(string fileName, int secondsBetweenPolls) - { - mySetRealSource( - new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, - ViewStreamDatabase.instance)); + private string _filename; + private IStreamMsgDeserilizer _deser = new StreamIODeserializationController(); + + + public LoaderStream(string filename) { + _filename = filename; } - public LoaderStream(string fileName) - { - if (fileName == null) - throw new LoaderStream.Exception("File name cannot be null"); - - for (int i = 0; i < 5; i++) - try - { - using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read)) - { - BinaryFormatter bf = new BinaryFormatter(); -#pragma warning disable SYSLIB0011 - BlueTable bt = (BlueTable)bf.Deserialize(fs); -#pragma warning restore SYSLIB0011 - mySetRealSource(new BlueRowCollectionStaticStream(bt)); - } - break; - } - catch (IOException) - { - System.Threading.Thread.Sleep(50); - } - + protected override void myOnFirstSubscriber() { + byte[] ser_msg = StreamSerializationUtils.read_from_file(_filename); + StreamIOMessage msg = _deser.deserialize(ser_msg); + Log.assert(msg.op == StreamIOMessage.StreamOp.CREATE); + IBlueRowCollection rows = msg.create_message.rows; + DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket); + myFireEvent(dp); } } } diff --git a/src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs b/src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs index 11bae65c1..1af59181b 100644 --- a/src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs @@ -1,64 +1,37 @@ -// using System; -// using System.Collections.Generic; -// using System.Text; -// using System.IO; +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; +using BCG; -// using System.Runtime.Serialization.Formatters.Binary; +namespace bts { + // this saves the total state in the filr provided in the cons + [DataStreamUserRespresentation("saver")] + public class SaverStream : ViewStream { + private ViewStream _stream; + private string _filename; + private IStreamIOMsgSerializer _serializer = new StreamIOSerializationController(); -// namespace BCG -// { -// [DataStreamUserRespresentation("saver")] -// public class SaverStream : ViewStream, ISideEffectStream -// { -// private ViewStream myStream; -// private string myFileName; + public SaverStream(ViewStream vs, string filename) { + _stream = vs; + _filename = filename; + } -// public class Exception : System.Exception -// { -// public Exception(string msg) : base(msg) { } -// } + protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); } -// public SaverStream(ViewStream vs, string fileName) -// { -// if (fileName == null) -// throw new SaverStream.Exception("File name cannot be null"); -// myStream = vs; -// myFileName = fileName; + void _handle_changes(object dp_obj) { + DataPacket dp = dp_obj as DataPacket; + _save(dp); + myFireEvent(dp_obj); + } -// } + protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); } -// protected override void myOnFirstSubscriber() -// { myStream.onChange += new ChangeDelegate(myHandleChanges); } + protected void _save(DataPacket dp) { + StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows); + byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg)); + StreamSerializationUtils.write_to_file(str_bytes, _filename); -// void myHandleChanges(object newValue) -// { -// DataPacket dp = newValue as DataPacket; -// mySave(dp); -// myFireEvent(newValue); -// } - -// protected override void myOnLastSubscriber() -// { myStream.onChange -= new ChangeDelegate(myHandleChanges); } - -// protected void mySave(DataPacket dp) -// { -// for (int i = 0; i < 5 ; i++) -// try -// { -// using (FileStream fs = new FileStream(myFileName, FileMode.Create, FileAccess.Write)) -// { -// BinaryFormatter bf = new BinaryFormatter(); -// BlueTable t = new BlueTable(dp.schema); -// foreach (IBlueRow r in dp.newRows) // to guarantee table is serializable -// t.addRow(new BlueRow(r)); -// bf.Serialize(fs, t); -// } -// break; -// } -// catch (IOException) -// { -// System.Threading.Thread.Sleep(5); -// } -// } -// } -// } + } + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs index 4b12556a1..4742545b1 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/TimeBasedFilter.cs @@ -1,37 +1,66 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; +using System.Text.RegularExpressions; using BCG.Utilities.General; -namespace BCG -{ - public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable - { +namespace BCG { + public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable { + + public static TimeSpan parse_duration(string input) { + int days = 0, hours = 0, minutes = 0, seconds = 0; + + // Regular expression to match the pattern (e.g., "1d", "1h", "15m", "45s") + Regex regex = new Regex(@"(\d+)([dhms])"); + MatchCollection matches = regex.Matches(input); + + foreach (Match match in matches) { + int value = int.Parse(match.Groups[1].Value); + string unit = match.Groups[2].Value; + + switch (unit) { + case "d": + days = value; + break; + case "h": + hours = value; + break; + case "m": + minutes = value; + break; + case "s": + seconds = value; + break; + } + } + + return new TimeSpan(days, hours, minutes, seconds); + } + #region constructors + + public TimeBasedFilter(string col_name, string row_time_to_live) + : this(col_name, parse_duration(row_time_to_live)) { + + } + public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive) - : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) - { + : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) { } public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) - : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) - { + : this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) { } public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive) - : this (vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) - { + : this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) { } - public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) - { - myVsd = vsd; - myRowTimeColumnName = rowTimeColumnName; - myRowTimeToLive = rowTimeToLive; - myFilterFrequency = filterFrequency; - myNextFilter = DateTime.MinValue; + public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) { + myVsd = vsd; + _time_col = rowTimeColumnName; + _ttl = rowTimeToLive; + myFilterFrequency = filterFrequency; + myNextFilter = DateTime.MinValue; // add worker myVsd.addWorker(this); @@ -41,32 +70,25 @@ namespace BCG public event FilterChangeDelegate onFilterChange; #region IStreamFilter - public bool filter(IBlueRow row) - { - DateTime now = DateTime.Now; - DateTime timeStamp = (DateTime)row[myRowTimeColumnName]; + public bool filter(IBlueRow row) { + DateTime now = DateTime.Now; - if (now > timeStamp) - { - TimeSpan span = now - timeStamp; - - if (span >= myRowTimeToLive) - { + DateTime row_time = row.get_datetime(_time_col, DateTime.Now); + if (now > row_time) { + TimeSpan span = now - row_time; + if (span >= _ttl) { return (false); } } - return (true); } #endregion #region IWorker - public void doWork() - { + public void doWork() { DateTime now = DateTime.Now; - if (now > myNextFilter) - { + if (now > myNextFilter) { // fire event onFilterChange(); @@ -76,8 +98,7 @@ namespace BCG #endregion #region IDisposable - public void Dispose() - { + public void Dispose() { // remove worker myVsd.removeWorker(this); } @@ -85,10 +106,20 @@ namespace BCG #region state private ViewStreamDatabase myVsd; - private string myRowTimeColumnName; - private TimeSpan myRowTimeToLive; - private TimeSpan myFilterFrequency; - private DateTime myNextFilter; + private string _time_col; + private TimeSpan _ttl; + private TimeSpan myFilterFrequency; + private DateTime myNextFilter; #endregion } + [DataStreamUserRespresentation("timefilter")] + public class TimeBasedFilterStream : ViewStream { + public TimeBasedFilterStream(ViewStream base_stream, string time_col, string time_span) + : + this(ViewStreamDatabase.instance, base_stream, time_col, TimeBasedFilter.parse_duration(time_span)) { } + public TimeBasedFilterStream(ViewStreamDatabase vsd, ViewStream base_stream, string time_col, TimeSpan time_span) { + TimeBasedFilter filter = new TimeBasedFilter(vsd, time_col, time_span); + mySetRealSource(new FilterStream(base_stream, filter)); + } + } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs b/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs index fa49a7f14..296154a75 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs @@ -1,51 +1,64 @@ using System; using System.Collections; +using System.Collections.Generic; + using BCG; using BCG.Utilities; -namespace BCG -{ - public class LiteralStream : ViewStream - { - public LiteralStream(string tableName, string schema, string value) - { - string [] columns = schema.Split(' '); +namespace BCG { + public class LiteralStream : ViewStream { + + public static List> parse_columns(string key_value_pairs) { + List> rv = new List>(); + string[] pairs = key_value_pairs.Split(' '); + foreach (string p in pairs) { + string[] kvp = p.Split(':'); + string col_name = kvp[0]; + string val = kvp[1]; + var tup = BlueDataUtility.guess_type(val); + var ci = new ColumnInfo(col_name, rv.Count, tup.Item1); + rv.Add(Tuple.Create(ci, tup.Item2)); + } + return rv; + } + + public LiteralStream(string tableName, string schema, string value) { + string[] columns = schema.Split(' '); - ColumnInfo [] ci = new ColumnInfo[columns.Length + 1]; - ci[0] = new ColumnInfo("id",0,typeof(int)); + ColumnInfo[] ci = new ColumnInfo[columns.Length + 1]; + ci[0] = new ColumnInfo("id", 0, typeof(int)); int n = 1; - foreach (string c in columns) - { - string [] columnAndType = c.Split('/'); + foreach (string c in columns) { + string[] columnAndType = c.Split('/'); if (columnAndType.Length == 1) - ci[n] = new ColumnInfo(c,n,typeof(double)); + ci[n] = new ColumnInfo(c, n, typeof(double)); else - ci[n] = new ColumnInfo(columnAndType[0],n,Type.GetType("System." + columnAndType[1],true)); + ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true)); n++; } - BlueTable t = new BlueTable(new BlueTableSchema(tableName,ci)); - string [] rows = value.Split(';'); + BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci)); + string[] rows = value.Split(';'); n = 0; - foreach (string r in rows) - try - { - if(r == string.Empty) + foreach (string r in rows) + try { + if (r == string.Empty) continue; t.addRow( - BlueDataUtility.createRowFromCSVFileRow (r, t.schema,++n)); - } - catch (Exception e) - { + BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n)); + } catch (Exception e) { throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message)); } mySetRealSource(new BlueRowCollectionStaticStream(t)); } - public LiteralStream(string schema, string value) : this ("literal",schema, value) - {} + public LiteralStream(string schema, string value) : this("literal", schema, value) { } + + // public LiteralStream(string value) { + + // } } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs b/src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs index ec6aaafd4..f637d3f11 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs @@ -2,8 +2,9 @@ using Bts.DataStream; using mbts.config; -namespace BCG -{ +using System; + +namespace BCG { /// @@ -19,19 +20,41 @@ namespace BCG protected IChangeableStreamMapper myChangeableMapper = null; private bool myUseNewMap = Configuration.get().UseNewMap; - public MapStream(ViewStream vs, IStreamMapper mapper) : base (vs) - { - if (myUseNewMap) - mySetRealSource (new SuperMapStream (vs, mapper)); - else - myMapper = mapper; + public class FuncWrapper : IStreamMapper { + private Func _schema_map_func; + Func _row_map_func; + + public FuncWrapper(Func schema_map_func, + Func row_map_func) { + _schema_map_func = schema_map_func; + _row_map_func = row_map_func; + } + public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { + return _schema_map_func(input_schema); + } + public IBlueRow map(IBlueRow row) { + return _row_map_func(row); + } + } - public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this (vs,(IStreamMapper)mapper) - { + + public MapStream(ViewStream vs, IStreamMapper mapper) : base(vs) { if (myUseNewMap) mySetRealSource(new SuperMapStream(vs, mapper)); else - { + myMapper = mapper; + } + + public MapStream(ViewStream vs, Func schema_map_func, + Func row_map_func) + : + this(vs, new FuncWrapper(schema_map_func, row_map_func)) { + } + + public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this(vs, (IStreamMapper)mapper) { + if (myUseNewMap) + mySetRealSource(new SuperMapStream(vs, mapper)); + else { mapper.onMapperChange += new MapperChangeDelegate(myMapperChanged); myChangeableMapper = mapper; } @@ -40,77 +63,68 @@ namespace BCG /// /// Gets or sets the value of the mapper. Setting the value will result in re /// - private void myMapperChanged () - { + private void myMapperChanged() { if (myLastArrivedPacket != null) - myChangeHandler (new DataPacket (myLastArrivedPacket.newRows, + myChangeHandler(new DataPacket(myLastArrivedPacket.newRows, myLastArrivedPacket.newRows, myLastArrivedPacket.newRows)); } - protected override void myHandleFirstCall () - { - - IBlueTableSchema outputSchema = myMapper.getOutputSchema (myLastArrivedPacket.newRows.schema); - myLastFiredPacket = new DataPacket (); - myLastFiredPacket.newRows = new BlueTable (outputSchema); - myLastFiredPacket.deletedRows = new BlueTable (outputSchema); + protected override void myHandleFirstCall() { + + IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema); + myLastFiredPacket = new DataPacket(); + myLastFiredPacket.newRows = new BlueTable(outputSchema); + myLastFiredPacket.deletedRows = new BlueTable(outputSchema); foreach (IBlueRow row in myLastArrivedPacket.newRows) - ((BlueTable)(myLastFiredPacket.newRows)).addRow (myMapper.map(row)); + ((BlueTable)(myLastFiredPacket.newRows)).addRow(myMapper.map(row)); myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows); } - protected override void myHandleIncrementalAdd() - { + protected override void myHandleIncrementalAdd() { //t.start (); - myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema); - foreach (IBlueRow row in myLastArrivedPacket.addedRows) - { - IBlueRow newRow = myMapper.map (row); - (myLastFiredPacket.newRows as BlueTable).addRow (newRow); - (myLastFiredPacket.addedRows as BlueTable).addRow (newRow); + myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows.schema); + foreach (IBlueRow row in myLastArrivedPacket.addedRows) { + IBlueRow newRow = myMapper.map(row); + (myLastFiredPacket.newRows as BlueTable).addRow(newRow); + (myLastFiredPacket.addedRows as BlueTable).addRow(newRow); } //t.stop (); //System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime); //System.Diagnostics.Trace.WriteLine (" "); } - protected override void myHandleIncrementalDelete() - { + protected override void myHandleIncrementalDelete() { //t.start (); - myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema); - foreach (IBlueRow row in myLastArrivedPacket.deletedRows) - { + myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows.schema); + foreach (IBlueRow row in myLastArrivedPacket.deletedRows) { IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]]; - (myLastFiredPacket.deletedRows as BlueTable).addRow (delRow); - (myLastFiredPacket.newRows as BlueTable).deleteRow (row); + (myLastFiredPacket.deletedRows as BlueTable).addRow(delRow); + (myLastFiredPacket.newRows as BlueTable).deleteRow(row); } //t.stop (); //System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime); } - - protected override void myOnFirstSubscriber() - { - myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler); + + protected override void myOnFirstSubscriber() { + myStream.onChange += new DataStream.ChangeDelegate(myChangeHandler); } - protected override void myOnLastSubscriber () - { - myInit (); + protected override void myOnLastSubscriber() { + myInit(); if (myStream != null) - myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler); + myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler); } - protected override void myOnTableClear () - { + protected override void myOnTableClear() { myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; - myLastFiredPacket.newRows = new BlueTable (myLastFiredPacket.deletedRows.schema); - myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema); + myLastFiredPacket.newRows = new BlueTable(myLastFiredPacket.deletedRows.schema); + myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema); } - + } /* diff --git a/src/DataStreamSuite/DataStream/Primitives/Map/SuperMapStream.cs b/src/DataStreamSuite/DataStream/Primitives/Map/SuperMapStream.cs index ee067b228..b4035e905 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Map/SuperMapStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Map/SuperMapStream.cs @@ -1,12 +1,13 @@ using System; using System.Collections.Generic; using System.Text; + using BCG.Utilities; using BCG.Utilities.General; + using mbts; -namespace BCG -{ +namespace BCG { /// /// SuperMap gives the user the ability /// to map one row to one or more rows @@ -17,13 +18,11 @@ namespace BCG /// can chage itself and raise an event causing /// a remapping to happen /// - public class SuperMapStream : ViewStream - { + public class SuperMapStream : ViewStream { #region super mapper that wraps around stream mapper - private class MyMappingWrapper : ISuperStreamMapper - { + private class MyMappingWrapper : ISuperStreamMapper { private IStreamMapper myMapper; - + public MyMappingWrapper(IStreamMapper mapper) { myMapper = mapper; } #region ISuperStreamMapper Members @@ -32,8 +31,7 @@ namespace BCG public IBlueRow[] map(IBlueRow inputRow) { return new IBlueRow[] { myMapper.map(inputRow) }; } - public event MapperChangeDelegate onMapperChange - { + public event MapperChangeDelegate onMapperChange { add { } remove { } } @@ -42,21 +40,18 @@ namespace BCG } #endregion - private class MyFuncWrapper : ISuperStreamMapper - { + private class MyFuncWrapper : ISuperStreamMapper { private Func _get_schema; private Func> _map_rows; public MyFuncWrapper(Func get_schema, - Func> map_rows) - { + Func> map_rows) { _get_schema = get_schema; _map_rows = map_rows; } public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); } - public IBlueRow[] map(IBlueRow r) - { + public IBlueRow[] map(IBlueRow r) { IEnumerable rows = _map_rows(r); return rows == null ? new IBlueRow[] { } : new List(rows).ToArray(); } @@ -76,19 +71,15 @@ namespace BCG public SuperMapStream(ViewStream vs, Func get_schema, Func> map_rows) - : this(vs, new MyFuncWrapper(get_schema, map_rows)) - { } + : this(vs, new MyFuncWrapper(get_schema, map_rows)) { } public SuperMapStream(ViewStream vs, IBlueTableSchema out_schema, Func> map_rows) - : this(vs, s => out_schema, map_rows) - { } + : this(vs, s => out_schema, map_rows) { } public SuperMapStream(ViewStream vs, IStreamMapper mapper) - : this(vs, new MyMappingWrapper(mapper)) - { } + : this(vs, new MyMappingWrapper(mapper)) { } - public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper) - { + public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper) { myStream = vs; myMapper = mapper; myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange); @@ -96,8 +87,7 @@ namespace BCG protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); } - protected override void myOnLastSubscriber() - { + protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleData); myRows = null; myBaseRows = null; @@ -105,8 +95,7 @@ namespace BCG //myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange); } - private void myHandleData(object newValue) - { + private void myHandleData(object newValue) { DataPacket idp = newValue as DataPacket; //HighPerformanceTimer timer = new HighPerformanceTimer(); //timer.start(); @@ -117,24 +106,19 @@ namespace BCG BlueTable dels = new BlueTable(myRows.schema), adds = new BlueTable(myRows.schema); - foreach (IBlueRow r in dp.deletedRows) - { + foreach (IBlueRow r in dp.deletedRows) { List mappedRows = myMap[(int)r["id"]]; - foreach (IBlueRow mr in mappedRows) - { + foreach (IBlueRow mr in mappedRows) { dels.addRow(mr); myRows.deleteRow(mr); } myMap.Remove((int)r["id"]); } - foreach (IBlueRow r in dp.addedRows) - { + foreach (IBlueRow r in dp.addedRows) { IBlueRow[] mappedRows = myMapper.map(r); - foreach (IBlueRow mr in mappedRows) - { + foreach (IBlueRow mr in mappedRows) { IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr; - if (!myMap.ContainsKey((int)r["id"])) - { + if (!myMap.ContainsKey((int)r["id"])) { myMap.Add((int)r["id"], new List()); } myMap[(int)r["id"]].Add(idChangedMapRow); @@ -145,13 +129,12 @@ namespace BCG //timer.stop(); // if (timer.lastIntervalTime > 20) // {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime)); - // myFireEvent(new DataPacket(myRows, adds, dels)); + myFireEvent(new DataPacket(myRows, adds, dels)); } - private void myHandleMapChange() - { + private void myHandleMapChange() { if (myBaseRows != null) - myHandleData(new DataPacket(myBaseRows, new BlueTable (myBaseRows), new BlueTable (myBaseRows))); + myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows))); } } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/DataPacketRecorderStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/DataPacketRecorderStream.cs new file mode 100644 index 000000000..77532d4ce --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/DataPacketRecorderStream.cs @@ -0,0 +1,142 @@ +using BCG; +using System; +using System.IO; +using MongoDB.Bson; +using BCG.Utilities.General; +using System.Linq; +using System.Collections.Generic; +using System.Runtime.Intrinsics.X86; + + +namespace bts { + // listens to the underlying stream, and writes each seriialized packet in a + // seperate file + [DataStreamUserRespresentation("dp_recorder")] + public class DataPacketRecorderStream : ViewStream { + private ViewStream _base_stream; + private readonly string _folder; + StreamIOSerializationController _serializer = new StreamIOSerializationController(); + private UInt32 _n_updates = 0; + public DataPacketRecorderStream(ViewStream base_stream, string folder) { + _base_stream = base_stream; + _folder = folder; + } + + protected override void myOnFirstSubscriber() { + _base_stream.onChange += _handle_update; + } + + protected override void myOnLastSubscriber() { + _base_stream.onChange -= _handle_update; + } + + private void _handle_update(object dp_obj) { + DataPacket dp = dp_obj as DataPacket; + byte[] ser_data = null; + if (_n_updates == 0) { + StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows); + ser_data = _serializer.serialize(new StreamIOMessage(cr_msg)); + } else { + StreamIOMessage.MultipleRemoveAddMessage mra = new StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows); + ser_data = _serializer.serialize(new StreamIOMessage(mra)); + } + Log.assert(ser_data != null); + _n_updates++; + string file_name = $"{_folder}\\{_n_updates}.bin"; + _write_to_file(ser_data, file_name); + myFireEvent(dp); + } + + private static void _write_to_file(byte[] data, string file_name) { + using (FileStream fs = new FileStream(file_name, FileMode.Create)) { + // Write the byte array to the file + fs.Write(data, 0, data.Length); + } + } + } + [DataStreamUserRespresentation("dp_replay")] + public class DataPacketReplayStream : ViewStream, IWorker { + private ViewStreamDatabase _vsd; + private readonly string _folder_path; + private readonly TimeSpan _fire_period; + IStreamMsgDeserilizer _deser = new StreamIODeserializationController(); + private int _file_ptr = 0; + private List> _file_data = new List>(); + private BlueTable _state = null; + + public DataPacketReplayStream(string folder_path, string fire_period) + : + this(ViewStreamDatabase.instance, folder_path, TimeBasedFilter.parse_duration(fire_period)) { + } + + public DataPacketReplayStream(ViewStreamDatabase vsd, string folder_path, TimeSpan fire_period) { + _vsd = vsd; + _folder_path = folder_path; + _fire_period = fire_period; + } + + protected override void myOnFirstSubscriber() { + MasterWorker.getInstance(false).addWorker(this, _fire_period); + _load_all_files(); + } + + protected override void myOnLastSubscriber() { + MasterWorker.getInstance(false).removeWorker(this); + } + + public void doWork() { + if (_file_ptr >= _file_data.Count()) + // we can set _file_ptr to 0 and keep going as well + return; + var curr_d = _file_data[_file_ptr++]; + var hext_txt = StreamSerializationUtils.to_hex(curr_d.Item2); + StreamIOMessage msg = _deser.deserialize(curr_d.Item2); + Log.Gen.info($"DataPacketReplayStream.doWork packet no : {_file_ptr} file : {curr_d.Item1} data: {hext_txt}"); + switch (msg.op) { + case StreamIOMessage.StreamOp.CREATE: { + _state = new BlueTable(msg.create_message.rows); + break; + } + case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: { + var mra = msg.multiple_remove_add_message; + foreach (IBlueRow r in mra.removed_rows) + _state.deleteRow(r); + foreach (IBlueRow r in mra.added_rows) + _state.addRow(r); + break; + } + } + var dp = BlueDataUtility.build_dp_inc(_state, myLastValue == null ? null : myLastValue as DataPacket, false); + myFireEvent(dp); + } + + private void _load_all_files() { + Func extract_file_name = (string full_path) => { + string fn = full_path.Split("\\").Last(); + fn = fn.Split(".").First(); + return fn; + + }; + string[] bin_files = Directory.GetFiles(_folder_path, "*.bin"); + + // Iterate over the .bin files and perform an action + foreach (string file in bin_files) { + byte[] file_data = File.ReadAllBytes(file); + _file_data.Add(Tuple.Create(file, file_data)); + } + _file_data.Sort((Tuple t1, Tuple t2) => { + string fn1 = extract_file_name(t1.Item1); + string fn2 = extract_file_name(t2.Item1); + int fn_i1 = Int32.Parse(fn1); + int fn_i2 = Int32.Parse(fn2); + if (fn_i1 < fn_i2) + return -1; + else if (fn_i1 == fn_i2) + return 0; + else if (fn_i1 > fn_i2) + return 1; + return 0; + }); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/SimpleTCPEchoStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/SimpleTCPEchoStream.cs new file mode 100644 index 000000000..555d9c471 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/SimpleTCPEchoStream.cs @@ -0,0 +1,55 @@ +namespace bts { + using BCG; + using BCG.Utilities.General; + + using System; + + public class SimpleTCPEchoStream : ViewStream, IWorker { + private string _ip; + private int _port; + private SimpleTCPClient _tcp_client; + private IBlueTableSchema _schema = BlueDataUtility.form_schema("SimpleTCPEchoStream", + Tuple.Create("id", TypeCode.Int32), + Tuple.Create("event", TypeCode.String)); + private BlueTable _table; + private SyncQueue _action_queue = new SyncQueue(); + + public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms) { + _ip = ip; + _port = port; + _table = new BlueTable(_schema); + _tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms); + } + + protected override void myOnFirstSubscriber() { + ViewStreamDatabase.instance.addWorker(this); + _tcp_client.register_listener((msg) => { _on_msg(msg); }); + _tcp_client.start(); + } + + private void _on_msg(string msg) { + _action_queue.enqueue(msg); + } + + public void doWork() { + if (_action_queue.getCount() == 0) + return; + BlueTable added_rows = new BlueTable(_schema); + var sz = _action_queue.getCount(); + for (int i = 0; i < sz; i++) { + string msg = _action_queue.dequeue(); + if (msg == null) + break; + IBlueRow nr = new BlueRow(_schema, _table.Count, msg); + _table.addRow(nr); + added_rows.addRow(nr); + } + if (added_rows.Count == 0) + return; + DataPacket dp = new DataPacket(_table, added_rows, new BlueTable(_schema)); + myFireEvent(dp); + } + + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs new file mode 100644 index 000000000..0cf9d6c7d --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs @@ -0,0 +1,77 @@ +using System.Net.Sockets; +using System.Net; + +using BCG; +using System; +using System.Collections.Generic; +using MongoDB.Bson; +using BCG.Utilities.General; +using System.Linq; + + +namespace bts { + // this class, a little specialized, takes the first snapshot thats fired + // from the base stream sorts them by a given column, and fires one row at a time + public class SnapshotReplayStream : ViewStream, IWorker { + + private ViewStreamDatabase _vsd; + private ViewStream _snapshot_stream; + private readonly string _sort_col; + private readonly int _buffer_size; + private TimeSpan _fire_period; + private int _in_row_ptr = 0; + private List _in_rows = new List(); + private CircularBufferTable _out_table; + + + public SnapshotReplayStream(ViewStream base_str, string sort_col, int buffer_size, string fire_period) + : this(ViewStreamDatabase.instance, base_str, sort_col, buffer_size, TimeBasedFilter.parse_duration(fire_period)) { } + + public SnapshotReplayStream(ViewStreamDatabase vsd, ViewStream base_str, string sort_col, int buffer_size, TimeSpan fire_period) { + _vsd = vsd; + _snapshot_stream = base_str; + _sort_col = sort_col; + _buffer_size = buffer_size; + _fire_period = fire_period; + } + + protected override void myOnFirstSubscriber() { + _snapshot_stream.onChange += _handle_stream_update; + MasterWorker.getInstance(false).addWorker(this, _fire_period); + } + + protected override void myOnLastSubscriber() { + base.myOnLastSubscriber(); + } + + private void _handle_stream_update(object dp_obj) { + DataPacket dp = dp_obj as DataPacket; + if (_in_rows.Count > 0) + return; + _out_table = new CircularBufferTable(dp.schema, _buffer_size); + // _in_rows is filled up only once + foreach (IBlueRow r in dp.newRows) { + _in_rows.Add(r); + } + _in_rows.Sort((IBlueRow r1, IBlueRow r2) => { + return BlueDataUtility.compare_rows(r1, r2, _sort_col); + }); + } + + public void doWork() { + if (_in_row_ptr >= _in_rows.Count) { + // we can loop around or not here + return; + } + if (_in_rows.Count > 0) { + // fire row + IBlueRow curr_row = _in_rows.ElementAt(_in_row_ptr++); + _out_table.add_row(curr_row); + DataPacket dp = BlueDataUtility.build_dp_inc(_out_table, + myLastValue == null ? null : myLastValue as DataPacket, false); + myFireEvent(dp); + } + } + } + +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs index 71f44e8ca..f58a7b829 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs @@ -56,8 +56,10 @@ namespace BCG { // _tests.Add(new Func(_run_full_stream_serialization_with_controller_test)); // _tests.Add(new Func(_run_pubsub_server_stream_tag_handler_test)); // _tests.Add(new Func(_run_single_tag_data_test_clean)); - _tests.Add(new Func(_cpp_data_test)); - _tests.Add(new Func(_write_updates_as_files)); + //_tests.Add(new Func(_cpp_data_test)); + //_tests.Add(new Func(_write_stream_updates_to_files)); + _tests.Add(new Func(_deserialize_pub_msg_test)); + } protected override void myOnFirstSubscriber() { @@ -143,7 +145,7 @@ namespace BCG { private TestResult _run_pubsub_client_msg_test() { PubSubClientMessage msg = new PubSubClientMessage() { - msg_type = PubSubMessageType.SUBSCRIIBE, + msg_type = PubSubMessageType.SUBSCRIBE, sub_msg = new bts.SubscribeMsg() { connection = Guid.NewGuid().ToString(), subscribing_app = Guid.NewGuid().ToString(), @@ -348,6 +350,17 @@ namespace BCG { }; } + private TestResult _deserialize_pub_msg_test() { + var pub_msg_ser = _read_from_file(@"C:\Temp\tests\pubsub_raw_msgs\publish_msg.bin"); + PubSubClientMessage ps_client_msg = new PubSubClientMessage(); + ps_client_msg.consruct_deserialize(pub_msg_ser); + bool passes = ps_client_msg.msg_type == PubSubMessageType.PUBLISH; + return new TestResult { + test_name = "_deserialize_pub_msg_test", + test_result = passes + }; + } + private TestResult _cpp_data_test() { //create and write schema IBlueTableSchema schema = BlueDataUtility.form_schema("test_table", @@ -437,7 +450,7 @@ namespace BCG { }; } - private TestResult _write_updates_as_files() { + private TestResult _write_stream_updates_to_files() { IBlueTableSchema schema = BlueDataUtility.form_schema("test_table", new Tuple("id", TypeCode.Int32), new Tuple("i_1", TypeCode.Int32), @@ -445,48 +458,54 @@ namespace BCG { new Tuple("d_1", TypeCode.Double), new Tuple("string_1", TypeCode.String) ); - var update_path = @"C:\Temp\tests\update_packets\"; + var packets_path = @"C:\Temp\tests\stream_packets\"; Func now_epochs = () => { DateTimeOffset now = DateTimeOffset.UtcNow; // Convert to Unix time in milliseconds long unix_time_milliseconds = now.ToUnixTimeMilliseconds(); return unix_time_milliseconds; }; - Func curr_file_path = () => { - var epoch_ms = now_epochs(); - var file_path = $"{update_path}{epoch_ms}.bin"; + Func curr_file_path = (id, create) => { + //var epoch_ms = now_epochs(); + var file_path = $"{packets_path}{id}.bin"; + if (create) + file_path = $"{packets_path}create_{id}.bin"; return file_path; }; // write 1000 stream updates { int init_sz = 100; BlueTable table = new BlueTable(schema); + StreamIOSerializationController ser_controller = new StreamIOSerializationController(); + // create table of 100 rows for (int i = 0; i < init_sz; i++) { - IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string"); + IBlueRow nr = new BlueRow(schema, i, i * 100, i * 200, i * 123.45, "test_string"); table.addRow(nr); } + // write create msg + StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table)); + byte[] create_ser_bytes = ser_controller.serialize(cr_msg); + var create_file_path = curr_file_path(table.Count, true); + _write_to_file(create_ser_bytes, create_file_path); + + // add 1000 rows and write an io msg for each update and write it to a file int n_updates = 1000; int period_ms = 50; - StreamIOSerializationController ser_controller = new StreamIOSerializationController(); + + // write a create for (int i = 0; i < n_updates; i++) { - if (i == 0) { - // creation - StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table)); - byte[] create_ser_bytes = ser_controller.serialize(cr_msg); - var file_path = curr_file_path(); - _write_to_file(create_ser_bytes, file_path); - } else { - // addition - BlueTable adds = new BlueTable(schema); - int id = table.Count; - IBlueRow nr = new BlueRow(schema, id, 100, 200, 123.45, "test_string"); - adds.addRow(nr); - BlueTable dels = new BlueTable(schema); - StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(dels, adds)); - byte[] mra_ser_bytes = ser_controller.serialize(mra_msg); - var file_path = curr_file_path(); - _write_to_file(mra_ser_bytes, file_path); - } + // addition + BlueTable adds = new BlueTable(schema); + int id = table.Count; + IBlueRow nr = new BlueRow(schema, id, i * 100, i * 200, i * 123.45, "test_string"); + adds.addRow(nr); + BlueTable dels = new BlueTable(schema); + table.addRow(nr); + StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(dels, adds)); + byte[] mra_ser_bytes = ser_controller.serialize(mra_msg); + var mra_file_path = curr_file_path(id, false); + _write_to_file(mra_ser_bytes, mra_file_path); + System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(period_ms)); } // now start writing increments and add 1 row at a time diff --git a/src/DataStreamSuite/DataStream/Primitives/MongoDB/BenzingaNewStream.cs b/src/DataStreamSuite/DataStream/Primitives/MongoDB/BenzingaNewStream.cs new file mode 100644 index 000000000..7660bcc66 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/MongoDB/BenzingaNewStream.cs @@ -0,0 +1,66 @@ +namespace bts { + using BCG; + + using System; + using System.Collections.Generic; + using System.Net; + using System.Text.Json; + using System.Text.Json.Nodes; + [DataStreamUserRespresentation("benzingaNews")] + public class BenzingaNewStream : ViewStream { + private IBlueTableSchema _schema = null; + private int _id = 0; + // constructor takes in the raw mongo db stream, and the time to live (ttl), "7d" for example + public BenzingaNewStream(ViewStream mongodb_raw_stream, string ttl) { + // Hiding body (with project, columns can be hidden, or renamed) + ViewStream output_stream = new ProjectStream(mongodb_raw_stream, "!body"); + // add column news_source = 'Benzinga', so we can append other news sources + output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga"); + // now SuperMapStream is used to map one row into multiple, one per symbol, + // (SuperMapStream takes 2 lambdas, one to get the enw schema and another to + // return an array of rows, per one row) + output_stream = new SuperMapStream(mongodb_raw_stream, + + (IBlueTableSchema s) => { + // this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange) + if (_schema == null) { + var out_schema = s; + out_schema = BlueDataUtility.remove_columns(out_schema, "securities"); + out_schema = BlueDataUtility.add_column(s, "doc_id_internal", typeof(string)); + out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string)); + out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string)); + out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool)); + _schema = out_schema; + } + return _schema; + }, + (IBlueRow r) => { + var json_objs = BlueDataUtility.json_to_json_node_list(r.get("securities")); + List expanded_rows = new List(); + if (json_objs.Count == 0) { + MutableBlueRow nr = new MutableBlueRow(_schema, r); + nr.setValue("id", _id++); + expanded_rows.Add(nr); + } + foreach (JsonNode jnode in json_objs) { + if (jnode is JsonObject json_object) { + MutableBlueRow nr = new MutableBlueRow(_schema, r); + nr.setValue("id", _id++); + nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}"); + nr.setValue("symbol", jnode["symbol"]); + nr.setValue("exchange", jnode["exchange"]); + nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString())); + expanded_rows.Add(nr); + } + } + return expanded_rows; + + }); + // now we creaate a TimeFilter stream to keep the table as a circular buffer of the last N days + // passed A an argument in the constructor above + // time_filter is a string like 7d 1h 30 m 45s + output_stream = new TimeBasedFilterStream(output_stream, "created_at", ttl); + mySetRealSource(output_stream); + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs index 43831d1ac..00dcd567e 100644 --- a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs @@ -1,7 +1,6 @@ +using BCG; using BCG.Utilities.General; -using bts; - using MongoDB.Bson; using MongoDB.Driver; @@ -9,14 +8,16 @@ using System; using System.Collections.Generic; using System.Diagnostics; -namespace BCG { +namespace bts { [DataStreamUserRespresentation("mongodb_sql")] public class MongoDBSQLStream : ViewStream, IWorker { private ViewStreamDatabase _vsd; private readonly string _connection_string, _db_name, _collection_name; private MongoClient _client; + // make this a sorted table private BlueTable _table; + // Access the database private IMongoDatabase _database; private IDictionary _id_map = new Dictionary(); @@ -38,7 +39,7 @@ namespace BCG { } protected override void myOnFirstSubscriber() { - MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); + MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10)); } protected override void myOnLastSubscriber() { @@ -133,7 +134,7 @@ namespace BCG { } protected override void myOnFirstSubscriber() { - MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); + MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10)); } protected override void myOnLastSubscriber() { @@ -154,7 +155,7 @@ namespace BCG { _table = _table == null ? null : new BlueTable(_table.schema); // Iterate through the documents and print values foreach (var document in documents.ToEnumerable()) { - bts.Log.Gen.info($"document: {document} json: {document.ToJson()}"); + //bts.Log.Gen.info($"document: {document} json: {document.ToJson()}"); if (_table == null) { _init_table(document); } @@ -207,4 +208,46 @@ namespace BCG { } } + [DataStreamUserRespresentation("mongodb_flat")] + public class MongoDBFlatStream : ViewStream { + public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration) + : + this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { } + + public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) { + ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd); + rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val"); + rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601); + //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration); + mySetRealSource(rs); + } + } + + + + [DataStreamUserRespresentation("newsreplay")] + public class NewsReplayStream : ViewStream { + public NewsReplayStream(string connection_string, string db_name, string collection_name, + int buffer_sz, string fire_period) : + this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { } + + public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, + int buffer_sz, TimeSpan fire_period) { + ViewStream r; + + r = new MongoDBStream(connection_string, db_name, collection_name, vsd); + r = new KeyValueToRelationalStream(r, + "doc_id", + "key", + "value"); + r = new DateStandardizationStream(r, + "created_at", + "ISO8601"); + r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period); + + mySetRealSource(r); + } + } + + } diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs b/src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs new file mode 100644 index 000000000..299b05426 --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs @@ -0,0 +1,172 @@ +namespace bts { + using BCG; + + using ProtoBuf; + + using System; + using System.Collections; + using System.Collections.Generic; + + public class ProjectData { + // schema from the base stream + public readonly IBlueTableSchema in_schema; + // this is the projected schema + public readonly IBlueTableSchema out_schema; + // projected cols to the input cols, for cols who's names are changed + public readonly IDictionary mapped_to_orig_col_map = new Dictionary(); + public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_, + IDictionary mapped_to_orig_col_map_) { + in_schema = in_schema_; + out_schema = out_schema_; + mapped_to_orig_col_map = mapped_to_orig_col_map_; + } + } + + public class NewProjectRow : IBlueRow { + #region IBlueRow members + private ProjectData _project_data; + private IBlueRow _row; + public NewProjectRow(ProjectData project_data, IBlueRow row) { + _project_data = project_data; + _row = row; + } + public IBlueTableSchema schema { get { return _project_data.out_schema; } } + #endregion + + #region ICollection members + public int Count { get { return schema.columnNames.Length; } } + + public bool IsSynchronized { get { return false; } } + + public object SyncRoot { get { return this; } } + + public void CopyTo(Array array, int index) { + int i = index; + foreach (object o in this as IEnumerable) + array.SetValue(o, i++); + } + #endregion + + #region IDictionary members + public bool IsFixedSize { get { return true; } } + + public bool IsReadOnly { get { return true; } } + + public object this[object key] { + get { + if (key is string) + return this[(string)key]; + return this[(int)key]; + } + set { throw new NotSupportedException("The IBlueRow is readonly"); } + } + + public object this[int index] { + get { + return this[schema.columnInfo[index].colName]; + } + } + + public object this[string col_name] { + get { + string mapped_to_col = null; + if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) { + return _row[mapped_to_col]; + } + return _row[col_name]; + } + } + + public ICollection Keys { get { return schema.columnNames; } } + + public ICollection Values { + get { + object[] retVal = new object[schema.columnNames.Length]; + CopyTo(retVal, 0); + return retVal; + } + } + + public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); } + + public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); } + + public bool Contains(object key) { return schema[(string)key] != null; } + + /// + /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to + /// iterate through the values, typecast this into an IEnumerable + /// + /// + public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } + + IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); } + + public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); } + #endregion + + } + + public class NewProjectStream : ViewStream, IStreamMapper { + + private ViewStream _stream; + private string _project_expression; + private ProjectData _project_data = null; + //public ProjectData + public NewProjectStream(ViewStream vs, string project_expression) { + _stream = vs; + _project_expression = project_expression; + _stream = new MapStream(vs, this); + mySetRealSource(_stream); + } + + public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { + _project_data = _create_project_data(input_schema, _project_expression); + return _project_data.out_schema; + } + + public IBlueRow map(IBlueRow row) { + return new NewProjectRow(_project_data, row); + } + + private static ProjectData _create_project_data(IBlueTableSchema input_schema, + string project_expression) { + project_expression = project_expression.Trim(); + string[] project_toks = project_expression.Split(' '); + HashSet removed_cols = new HashSet(); + Dictionary mapped_to_orig_map = new Dictionary(); + Dictionary orig_to_mapped = new Dictionary(); + foreach (string p_tok in project_toks) { + if (p_tok.StartsWith("!")) { + string removed_col = p_tok.Substring(1); + removed_cols.Add(removed_col); + } else if (p_tok.Contains(":")) { + string[] map_toks = p_tok.Split(":"); + string orig_col = map_toks[0]; + string mapped_col = map_toks[1]; + mapped_to_orig_map.Add(mapped_col, orig_col); + orig_to_mapped.Add(orig_col, mapped_col); + } else { + Log.Gen.error($"NewProjectStream unexpecetd token: {p_tok}"); + } + } + // make the new schema + List new_cols = new List(); + foreach (ColumnInfo ci in input_schema.columnInfo) { + if (removed_cols.Contains(ci.colName)) { + continue; + } else if (orig_to_mapped.ContainsKey(ci.colName)) { + string mapped_name = orig_to_mapped[ci.colName]; + ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType); + new_cols.Add(nc); + } else { + ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType); + new_cols.Add(nc); + } + } + IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray()); + return new ProjectData(input_schema, new_schema, mapped_to_orig_map); + } + + } +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs index 8667ca2cf..6a8be6cb5 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs @@ -5,6 +5,7 @@ namespace BCG { using System.Collections.Generic; using System.Linq; + [DataStreamUserRespresentation("oldproject")] public class ProjectStream : ViewStream { private ProjectHelper myHelper; diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs index 7fe43313c..6c68fd2dd 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs @@ -3,24 +3,23 @@ using System.Collections.Generic; using System.Text; -namespace BCG -{ +namespace BCG { /// /// If one adds annother value to this enum, be sure to update _date_conversion_to_type_map below as well /// - public enum DateConversion - { + public enum DateConversion { ConvertDoubleToDateTime = 0, ChopTime, UTCToCurrent, ConvertStringToDateTime, ConvertDateTimeToMilliseconds, - SecondsOfDay + SecondsOfDay, + ISO8601, + EpochToDatetime } [DataStreamUserRespresentation("dateStandardize")] - public class DateStandardizationStream : ViewStream, IStreamMapper - { + public class DateStandardizationStream : ViewStream, IStreamMapper { private IBlueTableSchema mySchema; private DateConversion myConversion; private string myConversionColName; @@ -31,12 +30,13 @@ namespace BCG {DateConversion.UTCToCurrent , typeof(DateTime)}, {DateConversion.ConvertStringToDateTime , typeof(DateTime)}, {DateConversion.ConvertDateTimeToMilliseconds, typeof(Double)}, - {DateConversion.SecondsOfDay , typeof(Double)} + {DateConversion.SecondsOfDay , typeof(Double)}, + {DateConversion.ISO8601 , typeof(DateTime)}, + {DateConversion.EpochToDatetime , typeof(DateTime)} }; - private static DateConversion _parse(string s) - { + private static DateConversion _parse(string s) { DateConversion rv = DateConversion.ChopTime; Enum.TryParse(s, false, out rv); return rv; @@ -45,11 +45,9 @@ namespace BCG /// for DSQA /// public DateStandardizationStream(ViewStream vs, string colName, string conversion) - : this(vs, colName, _parse(conversion)) - { } + : this(vs, colName, _parse(conversion)) { } - public DateStandardizationStream(ViewStream vs, string colName, DateConversion conversion) - { + public DateStandardizationStream(ViewStream vs, string colName, DateConversion conversion) { myConversionColName = colName; myConversion = conversion; mySetRealSource(new MapStream(vs, this)); @@ -57,24 +55,18 @@ namespace BCG #region IStreamMapper Members - public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) - { - if (mySchema == null) - { + public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) { + if (mySchema == null) { List cis = new List(); - foreach (ColumnInfo c in inputSchema.columnInfo) - { - if (c.colName == myConversionColName) - { + foreach (ColumnInfo c in inputSchema.columnInfo) { + if (c.colName == myConversionColName) { Type convType; - if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType)) - { + if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType)) { //BCG.Utilities.BlueEventLog.logEvent("Please update the _date_conversion_to_type_map in DateStandardizationStream.cs.", BCG.Utilities.BlueEventLevel.Error); convType = typeof(DateTime); } cis.Add(new ColumnInfo(c.colName, c.colNo, convType)); - } - else + } else cis.Add(new ColumnInfo(c.colName, c.colNo, c.colType)); } mySchema = new BlueTableSchema(inputSchema.tableName, cis.ToArray()); @@ -86,56 +78,59 @@ namespace BCG #region IDynamicStreamMapper Members - public IBlueRow map(IBlueRow row) - { - return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName); + public IBlueRow map(IBlueRow row) { + MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row); + rv.setValue(myConversionColName, mapDateTime(row[myConversionColName], myConversion)); + return rv; + //return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName); } #endregion - public static object mapDateTime(object objIn, DateConversion conv) - { - if (objIn == DBNull.Value || objIn == null) - { + public static object mapDateTime(object obj_in, DateConversion conv) { + if (obj_in == DBNull.Value || obj_in == null) { return DBNull.Value; - } - else - { - if (conv == DateConversion.ChopTime) - { - DateTime dtIn = (DateTime)objIn; - return dtIn.Date; - } - else if (conv == DateConversion.ConvertDoubleToDateTime) - { - double dtIn = (double)objIn; - return DateTime.FromOADate(dtIn); - } - else if (conv == DateConversion.UTCToCurrent) - { - DateTime dtIn = (DateTime)objIn; - return dtIn.ToLocalTime(); - } - else if (conv == DateConversion.ConvertStringToDateTime) - { - return DateTime.Parse((string)objIn); - } - else if (conv == DateConversion.ConvertDateTimeToMilliseconds) - { - DateTime dtIn = (DateTime)objIn; - return dtIn.TimeOfDay.TotalMilliseconds; - } - else if (conv == DateConversion.SecondsOfDay) - { - DateTime dtIn = (DateTime)objIn; - DateTime start_of_day = dtIn.Date; - TimeSpan day_period = dtIn - start_of_day; - return day_period.TotalSeconds; - } - else - { - bts.Log.Gen.info("incorrect type"); - return null; + } else { + switch (conv) { + case DateConversion.ChopTime: { + DateTime dtIn = (DateTime)obj_in; + return dtIn.Date; + } + case DateConversion.ConvertDoubleToDateTime: { + double dtIn = (double)obj_in; + return DateTime.FromOADate(dtIn); + } + case DateConversion.UTCToCurrent: { + DateTime dtIn = (DateTime)obj_in; + return dtIn.ToLocalTime(); + } + case DateConversion.ConvertStringToDateTime: { + return DateTime.Parse((string)obj_in); + } + case DateConversion.ConvertDateTimeToMilliseconds: { + DateTime dtIn = (DateTime)obj_in; + return dtIn.TimeOfDay.TotalMilliseconds; + } + case DateConversion.SecondsOfDay: { + DateTime dtIn = (DateTime)obj_in; + DateTime start_of_day = dtIn.Date; + TimeSpan day_period = dtIn - start_of_day; + return day_period.TotalSeconds; + } + case DateConversion.ISO8601: { + string iso8601Date = obj_in.ToString(); + return DateTime.Parse(iso8601Date, null, System.Globalization.DateTimeStyles.RoundtripKind); + } + case DateConversion.EpochToDatetime: { + double d = Convert.ToDouble(obj_in); + long unix_time_seconds = (long)d; + DateTime epoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + return epoch.AddSeconds(unix_time_seconds); + } + default: { + bts.Log.Gen.info("incorrect type"); + return DBNull.Value; + } } } } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs index 9f5af7beb..9422271aa 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs @@ -6,7 +6,6 @@ using System.Reflection; using System.IO; using System.Diagnostics; using System.Linq; -using System.Globalization; //using BCG.Utilities; using BCG.Utilities.General; @@ -14,7 +13,8 @@ using System.Linq.Expressions; //using mbts.spark; using mbts; using BCG.Extensions; -using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.Json.Nodes; namespace BCG { public static class BlueDataUtility { @@ -49,12 +49,6 @@ namespace BCG { rv.Add(r.get(col)); return rv; } - public static IEnumerable get_colvalues(IBlueRowCollection rows, string col) { - List rv = new List(); - foreach (IBlueRow r in rows) - rv.Add(r.get(col)); - return rv; - } public static IDictionary> group_by(IBlueRowCollection rows, string gp_col) { Dictionary> rv = new Dictionary>(); @@ -74,12 +68,17 @@ namespace BCG { // The values are returned as string values // doc_col is the column specifyng the doc name (following mongoDB's nomenclature, one set of key value rows ar a 'doc') public static IBlueRowCollection keyvalue_to_relational(IBlueRowCollection kv_rows, string doc_col, string key_col, string val_col) { - HashSet unique_cols = new HashSet(); + HashSet keys = new HashSet(); foreach (string col in get_column_values(kv_rows, key_col)) { - unique_cols.Add(col); + keys.Add(col); } - //unique_cols.Add(doc_col); - var schema_cols = unique_cols.Select(c => Tuple.Create(c, TypeCode.String)).ToList(); + if (keys.Contains("id")) { + keys.Remove("id"); + } + //unique_cols.Add(doc_col) + // all the keys become cols + var schema_cols = keys.Select(c => Tuple.Create(c, TypeCode.String)).ToList(); + //schema of the final table needs doc and id schema_cols.Insert(0, Tuple.Create(doc_col, BlueDataUtility.get_typecode(typeof(T)))); schema_cols.Insert(0, Tuple.Create("id", TypeCode.Int32)); var schema = BlueDataUtility.form_schema("relational", schema_cols.ToArray()); @@ -91,6 +90,8 @@ namespace BCG { nr.setValue(doc_col, kv.Key); foreach (var r in kv.Value) { var col = r.get(key_col); + if (!keys.Contains(col)) + continue; var val = r.get(val_col); nr.setValue(col, val); } @@ -188,6 +189,35 @@ namespace BCG { #endregion + #region json + + public static List json_to_json_node_list(string jsonString) { + var jsonNodeList = new List(); + bool noe = String.IsNullOrEmpty(jsonString); + if (noe) + return jsonNodeList; + try { + // Parse the JSON string + JsonArray jsonArray = JsonNode.Parse(jsonString) as JsonArray; + + if (jsonArray == null) { + Console.WriteLine("The JSON string must represent a list."); + return jsonNodeList; + } + + // Iterate through each JsonNode in the JsonArray + foreach (JsonNode jsonNode in jsonArray) { + jsonNodeList.Add(jsonNode); + } + } catch (JsonException ex) { + Console.WriteLine($"Invalid JSON: {ex.Message}"); + } + + return jsonNodeList; + } + + #endregion + public static void print_table(this IBlueRowCollection table, Func get_col_width) { var rv = get_printed_rows(table, get_col_width); Console.WriteLine(rv); @@ -276,6 +306,39 @@ namespace BCG { #endregion + public static Tuple guess_type(string value) { + if (string.IsNullOrEmpty(value)) { + return Tuple.Create(typeof(string), (object)value); + } + + int val = 0; + // Try to parse as integer + if (int.TryParse(value, out val)) { + return Tuple.Create(typeof(int), (object)val); + } + + double dval = 0.0; + // Try to parse as double + if (double.TryParse(value, out dval)) { + return Tuple.Create(typeof(double), (object)dval); + } + + DateTime dt; + // Try to parse as DateTime + if (DateTime.TryParse(value, out dt)) { + return Tuple.Create(typeof(DateTime), (object)dt); + } + + bool bval; + // Try to parse as boolean + if (bool.TryParse(value, out bval)) { + return Tuple.Create(typeof(bool), (object)bval); + } + + // If none of the above, return string type + return Tuple.Create(typeof(string), (object)value); + } + public static IBlueTableSchema form_schema(string table_name, params Tuple[] cols) { List cis = new List(); foreach (Tuple c in cols) @@ -285,165 +348,6 @@ namespace BCG { return new BlueTableSchema(table_name, cis.ToArray()); } - public static TypeCode convert_column_type(IEnumerable col_vals) { - // bool - bool success = true; - foreach (var val in col_vals) { - bool parsed_val; - success = success && Boolean.TryParse(val, out parsed_val); - if (!success) - break; - } - if (success) - return TypeCode.Boolean; - // int - success = true; - foreach (var val in col_vals) { - int parsed_val; - success = success && Int32.TryParse(val, out parsed_val); - if (!success) - break; - } - if (success) - return TypeCode.Int32; - // try double - success = true; - foreach (var val in col_vals) { - double parsed_val; - success = success && Double.TryParse(val, out parsed_val); - if (!success) - break; - } - if (success) - return TypeCode.Double; - // try DateTime - success = true; - foreach (var val in col_vals) { - DateTime parsed_val; - success = success && DateTime.TryParse(val, out parsed_val); - if (!success) - break; - } - if (success) - return TypeCode.DateTime; - bts.Log.assert(false); - return TypeCode.Empty; - } - - // public static Type guess_type(string val) { - // //DateTime - // { - // DateTime dt; - // var success = DateTime.TryParseExact(val, "yyyy-MM-dd HH:mm:ss", null, DateTimeStyles.AllowWhiteSpaces, out dt); - // if (success) { - // return typeof(DateTime); - // } - // DateTimeOffset dtoff; - // success = DateTimeOffset.TryParseExact(val, "yyyy-MM-dd HH:mm:sszzz", - // null, System.Globalization.DateTimeStyles.None, out dtoff); - - // if (success) { - // return typeof(DateTime); - // } - // } - // //int - // { - // int i; - // var success = Int32.TryParse(val, out i); - // if (success) { - // return typeof(Int32); - // } - // } - // //double - // { - // double d; - // var success = Double.TryParse(val, out d); - // if (success) { - // return typeof(Double); - // } - // } - // //bool - // { - // bool b; - // var success = Boolean.TryParse(val, out b); - // if (success) { - // return typeof(bool); - // } - // } - // //string - // return typeof(string); - // } - - private static Type[] _parsable_types = { typeof(DateTime), typeof(int), typeof(double), typeof(bool) }; - public static Type guess_type(string val) { - foreach (var t in _parsable_types) { - var type_val = convert_to_type(t, val); - if (type_val != null) - return t; - } - return typeof(string); - - } - - public static IBlueTableSchema get_typed_schema(IBlueRowCollection rows) { - // just use one row to guess - List cis = new List(); - foreach (IBlueRow r in rows) { - foreach (var ci in r.schema.columnInfo) { - if (ci.colType != typeof(System.String)) { - cis.Add(ci); - continue; - } - Type type = guess_type(r[ci.colName] as string); - cis.Add(new ColumnInfo(ci.colName, ci.colNo, type)); - } - return new BlueTableSchema(cis.ToArray(), rows.schema.tableName); - } - return null; - } - - static object convert_to_type(Type t, string value) { - // Attempt to convert the string to the specified type - try { - return Convert.ChangeType(value, t); - } catch (Exception) { - if (t == typeof(double)) { - // some doubles come with currency and ',' we clean that up here (there might be better ways to do this) - string cleaned_val = new string(value.Where(c => c != '$' && c != ',').ToArray()); - cleaned_val = cleaned_val.Replace('−', '-'); - double rv_d; - if (Double.TryParse(cleaned_val, out rv_d)) - return rv_d; - else - return null; - } - return null; - // Handle conversion failure gracefully - // throw new ArgumentException("Failed to convert the string to the specified type."); - } - } - - // converts a row where all columns are string types to a schema where the best types are guesed - public static IBlueRowCollection get_typed_table(IBlueRowCollection string_rows) { - Func get_typed_row = (IBlueRow string_row, IBlueTableSchema typed_schema) => { - MutableBlueRow rv = new MutableBlueRow(typed_schema); - foreach (ColumnInfo ci in typed_schema.columnInfo) { - var typed_val = convert_to_type(ci.colType, value: string_row[ci.colName].ToString()); - rv.setValue(ci.colName, typed_val); - } - return rv; - }; - IBlueTableSchema typed_schema = get_typed_schema(string_rows); - var rv = new BlueTable(typed_schema); - foreach (IBlueRow r in string_rows) { - var typed_row = get_typed_row(r, typed_schema); - rv.addRow(typed_row); - } - return rv; - } - - - /// /// Gets the Hashcode of the row with the boolean indicating whether the order of the rows should /// be alphabetic or not @@ -2297,33 +2201,5 @@ namespace BCG { } #endregion - - public class RowComparer : IComparer { - private readonly IEnumerable> _sort_cols; - public RowComparer(IEnumerable> sort_cols) { - _sort_cols = sort_cols; - } - public int Compare(IBlueRow? x, IBlueRow? y) { - Func compare_cols = (o1, o2) => { - return (o1 as IComparable).CompareTo(o2); - }; - foreach (var c in _sort_cols) { - int i = compare_cols(x[c.Item1], y[c.Item1]); - - } - return 1; - } - - } - public static IList get_sorted_row(IBlueRowCollection rows, IEnumerable> sort_cols) { - List row_list = new List(); - foreach (IBlueRow r in rows) { - row_list.Add(r); - } - var row_comparer = new RowComparer(sort_cols); - ParallelSort.QuicksortSequential(row_list, row_comparer); - return row_list; - - } } } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs new file mode 100644 index 000000000..ef67a2264 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Data; +using System.Linq; + +using BCG; + +namespace bts { + public class CircularBufferTable : IBlueRowCollection { + private BlueTable _table; + private List _ordered_list = new List(); + private int _buffer_sz; + public CircularBufferTable(IBlueTableSchema schema, int buffer_sz) { + _table = new BlueTable(schema); + _buffer_sz = buffer_sz; + } + + // public OrderedBlueTable(List rows, int buffer_sz) { + // _buffer_sz = buffer_sz; + // foreach (IBlueRow r in rows) + // add_row(r); + // } + + // returns the row that might need to be deleted to keep + // the circular buffer intact + public IBlueRow add_row(IBlueRow row) { + _ordered_list.Add(row); + _table.addRow(row); + if (_ordered_list.Count > _buffer_sz) { + IBlueRow rv = _ordered_list.ElementAt(0); + _ordered_list.RemoveAt(0); + _table.deleteRow(rv); + return rv; + } + return null; + } + + public IBlueRow this[int id] { + get { + return _table[id]; + } + } + public IBlueTableSchema schema { + get { + return _table.schema; + } + } + public bool containsID(int id) { + return _table.containsID(id); + } + + public void CopyTo(Array array, int index) { _table.CopyTo((IBlueRow[])array, index); } + public int Count { get { return _table.Count; } } + public bool IsSynchronized { get { return false; } } + public object SyncRoot { get { return this; } } + public System.Collections.IEnumerator GetEnumerator() { return _table.GetEnumerator(); } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs index 1965d8b57..7e97f13e3 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs @@ -33,6 +33,7 @@ namespace bts { DateTime dt = r.get(col_no); DateTimeOffset dt_off = new DateTimeOffset (dt); long mseconds = dt_off.ToUnixTimeMilliseconds(); + //Log.assert(mseconds < long.MaxValue); bw.Write(mseconds); } }, @@ -55,9 +56,14 @@ namespace bts { {typeof(char), (br) => {return br.ReadChar();}}, // TODO DateTime {typeof(DateTime), (br) => { + try{ long msecs = br.ReadInt64(); DateTimeOffset dt_off = DateTimeOffset.FromUnixTimeMilliseconds(msecs); return dt_off.DateTime; + } + catch { + return DateTime.MinValue; + } }}, //{typeof(byte), (br) => {return br.ReadByte()} {typeof(Decimal), (br) => {return br.ReadDecimal();}}, @@ -192,7 +198,19 @@ namespace bts { return deser.deserialize(ref br); } - //public static void update(this IPubSubServerTagHandler tag_handler, ) + public static void write_to_file(byte[] data, string file_name) { + if (String.IsNullOrEmpty(file_name)) + return; + using (FileStream fs = new FileStream(file_name, FileMode.Create)) { + // Write the byte array to the file + fs.Write(data, 0, data.Length); + } + } + + public static byte[] read_from_file(string file_path) { + var rv = File.ReadAllBytes(file_path); + return rv; + } } } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/Deserialization/UniversalRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/Deserialization/UniversalRowDeserializer.cs index 58fa6a44a..960fe12ac 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/Deserialization/UniversalRowDeserializer.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/Deserialization/UniversalRowDeserializer.cs @@ -23,6 +23,7 @@ namespace bts { public IBlueRow deserialize_row(ref BinaryReader br) { byte[] null_map = br.ReadBytes(_schema.columnInfo.Length); List row_vals = new List(); + List ptrs = new List(); foreach (ColumnInfo ci in _schema.columnInfo) { var idx = ci.colNo; bool is_dbnull = null_map[ci.colNo] == StreamSerializationUtils.row_null_value_flag; @@ -30,6 +31,7 @@ namespace bts { row_vals.Add(DBNull.Value); } else { var val = StreamSerializationUtils.deserialize_column(ref br, ci.colType); + ptrs.Add(br.BaseStream.Position); row_vals.Add(val); } } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/PubSubServer/PubsubServerStreamTagHandler.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/PubSubServer/PubsubServerStreamTagHandler.cs index e768cc3fd..825b7506e 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/PubSubServer/PubsubServerStreamTagHandler.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/PubSubServer/PubsubServerStreamTagHandler.cs @@ -59,7 +59,8 @@ namespace bts { return null; }; var rv = get_content_(); - Log.Gen.info("PubsubServerStreamTagHandler.get_content rv: {rv == null ? null : rv.Length}"); + var rv_txt = rv == null ? "null" : rv.Length.ToString(); + Log.Gen.info($"PubsubServerStreamTagHandler.get_content rv: {rv_txt}"); return rv; } public string[] supportedTagTypes { diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs index 26f1934d4..d25875479 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Program.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs @@ -67,7 +67,7 @@ static ViewStream? build_stream(string? stream_expression) { static void start_echo_client() { Console.WriteLine("Starting DSPublisher in TCP client echo mode, Hit key to continue"); Console.ReadLine(); - SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345); + SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345, 1000, 1000); client.start(); while (true) { System.Threading.Thread.Sleep(1000); @@ -78,7 +78,7 @@ static void start_echo_client() { void _publish_streams(IEnumerable publishers) { PubTableBroker pb_broker = new PubTableBroker(); - CloudStreamManager csm = new CloudStreamManager(); + RemoteQueryManager csm = new RemoteQueryManager(); var csvPublishers = new List(); foreach (var publisher in publishers) { @@ -96,6 +96,7 @@ void _publish_streams(IEnumerable publishers) { ViewStreamDatabase.instance.doWork(); csm.doWork(); MasterWorker.getInstance(false).doWork(); + ManagedSubscriber.instance.doWork(); } ViewPublisher? createViewPublisher(PublishedStream ps) { diff --git a/src/DataStreamSuite/DataStreamTests/PubSubMessages/PuSubMessagesTests.cs b/src/DataStreamSuite/DataStreamTests/PubSubMessages/PuSubMessagesTests.cs index 0610682bd..a7ad5f9cd 100644 --- a/src/DataStreamSuite/DataStreamTests/PubSubMessages/PuSubMessagesTests.cs +++ b/src/DataStreamSuite/DataStreamTests/PubSubMessages/PuSubMessagesTests.cs @@ -111,7 +111,7 @@ namespace bts { [Test] public static void test_PubSubClientMessage() { PubSubClientMessage msg = new PubSubClientMessage() { - msg_type = PubSubMessageType.SUBSCRIIBE, + msg_type = PubSubMessageType.SUBSCRIBE, unsub_msg = null, transaction_msg = null, pub_msg = null, diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs b/src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs index a2b4c724b..0aa0c6a02 100644 --- a/src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs +++ b/src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs @@ -171,7 +171,7 @@ namespace bts { } public enum PubSubMessageType { - SUBSCRIIBE = 0, + SUBSCRIBE = 0, UNSUBSCRIIBE, TRANSACTION, PUBLISH @@ -189,7 +189,7 @@ namespace bts { public PubSubClientMessage(SubscribeMsg sub_msg_) { sub_msg = sub_msg_; - msg_type = PubSubMessageType.SUBSCRIIBE; + msg_type = PubSubMessageType.SUBSCRIBE; } public PubSubClientMessage(UnsubscribeMsg unsub_msg_) { @@ -211,7 +211,7 @@ namespace bts { BinaryWriter bw = new BinaryWriter(str); bw.Write((int)msg_type); switch (msg_type) { - case PubSubMessageType.SUBSCRIIBE: + case PubSubMessageType.SUBSCRIBE: sub_msg.serialize_into_str(str); break; case PubSubMessageType.UNSUBSCRIIBE: @@ -240,7 +240,7 @@ namespace bts { int type = br.ReadInt32(); msg_type = (PubSubMessageType)type; switch (msg_type) { - case PubSubMessageType.SUBSCRIIBE: + case PubSubMessageType.SUBSCRIBE: sub_msg = new SubscribeMsg(); sub_msg.consruct_deserialize(str); break; @@ -268,7 +268,7 @@ namespace bts { public override string ToString() { var rv = $"type: {msg_type} "; switch (msg_type) { - case PubSubMessageType.SUBSCRIIBE: + case PubSubMessageType.SUBSCRIBE: rv += $"{sub_msg}"; break; case PubSubMessageType.UNSUBSCRIIBE: diff --git a/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/ManagedSubscriber.cs b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/ManagedSubscriber.cs index 93211605a..c52b0d9b9 100644 --- a/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/ManagedSubscriber.cs +++ b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/ManagedSubscriber.cs @@ -1,114 +1,116 @@ using System; -using System.Collections; -using System.IO; using System.Collections.Generic; +using System.IO; +using System.Linq; -using BCG.Utilities; using BCG.Utilities.General; +using OnData = System.Action; + namespace BCG { - public delegate void Handler(IPubItem pi); - public delegate void RawDataHandler(string tag, Stream s); + // public delegate void Handler(IPubItem pi); + // public delegate void RawDataHandler(string tag, Stream s); /// /// Provides a callback based interface to the SubscriptionCore (this class is mainly here to allow backward compatibility /// with PubSub, as some clients are still using ManagedSubcribers) /// public class ManagedSubscriber : IWorker, IDisposable { - private static ManagedSubscriber mySelf; - private SubscriberCore mySubsCore; - private IDictionary myTypedDispatchTable = new Hashtable(), - myRawDispatchTable = new Hashtable(); + private static ManagedSubscriber _self; + private SubscriberCore _subcore; + // private IDictionary myTypedDispatchTable = new Hashtable(), + // myRawDispatchTable = new Hashtable(); + private Dictionary> _dispatch_table = new Dictionary>(); public ManagedSubscriber() : this(new SubscriberCore()) { } - public ManagedSubscriber(SubscriberCore subsCore) { mySubsCore = subsCore; } + public ManagedSubscriber(SubscriberCore subsCore) { _subcore = subsCore; } public void doWork() { var msg_map = new Dictionary>(); - var updates = mySubsCore.get_updates(TimeSpan.Zero); - foreach (bts.PubSubServerMessage ps_msg in updates) { - List tag_list = null; - if (!msg_map.TryGetValue(ps_msg.tag, out tag_list)) { - tag_list = new List(); - tag_list.Add(ps_msg); - } - tag_list.Add(ps_msg); + var updates = _subcore.get_updates(TimeSpan.Zero); + if (updates.Count() == 0) + return; + _fire_events(updates); + } + + public void subscribe(string tag, OnData on_data) { + List tag_cbs = null; + if (!_dispatch_table.TryGetValue(tag, out tag_cbs)) { + tag_cbs = new List(); + _dispatch_table.Add(tag, tag_cbs); + _subcore.subscribe(tag); ; } - myFireEvents(msg_map); + tag_cbs.Add(on_data); } - public void subscribe(string tag, Handler handler) { - myTypedDispatchTable[tag] = !myTypedDispatchTable.Contains(tag) ? - new TypedPubItemHandler(tag, mySubsCore) : myTypedDispatchTable[tag]; - (myTypedDispatchTable[tag] as TypedPubItemHandler).addSubscription(handler); + // public void subscribe(string tag, Handler handler) { + // myTypedDispatchTable[tag] = !myTypedDispatchTable.Contains(tag) ? + // new TypedPubItemHandler(tag, mySubsCore) : myTypedDispatchTable[tag]; + // (myTypedDispatchTable[tag] as TypedPubItemHandler).addSubscription(handler); + // } + + // public void subscribe(string tag, Handler handler, IGenericFormatter formatter) { + // myTypedDispatchTable[tag] = !myTypedDispatchTable.Contains(tag) ? + // new TypedPubItemHandler(tag, mySubsCore, formatter) : myTypedDispatchTable[tag]; + // (myTypedDispatchTable[tag] as TypedPubItemHandler).addSubscription(handler); + // } + + // public void subscribe(string tag, RawDataHandler handler) { + // myRawDispatchTable[tag] = !myRawDispatchTable.Contains(tag) ? + // new RawPubItemHandler(tag, mySubsCore) : myRawDispatchTable[tag]; + // (myRawDispatchTable[tag] as RawPubItemHandler).addSubscription(handler); + // } + + // public void unsubscribe(string tag, Handler handler) { + // if (myTypedDispatchTable.Contains(tag)) { + // (myTypedDispatchTable[tag] as TypedPubItemHandler).removeSubscription(handler); + // if ((myTypedDispatchTable[tag] as SingleSubHandler).nSubscribers == 0) + // myTypedDispatchTable.Remove(tag); + // } else { } + // // BlueEventLog.logEvent("Trying to unsubscribe a PubItem tag that was never subsribed", BlueEventLevel.Error); + // } + + public void unsubcribe(string tag, OnData on_data) { + List tag_cbs = null; + if (_dispatch_table.TryGetValue(tag, out tag_cbs)) { + tag_cbs.Remove(on_data); + if (tag_cbs.Count == 0) { + _dispatch_table.Remove(tag); + _subcore.unsubscribe(tag); + } + } } - public void subscribe(string tag, Handler handler, IGenericFormatter formatter) { - myTypedDispatchTable[tag] = !myTypedDispatchTable.Contains(tag) ? - new TypedPubItemHandler(tag, mySubsCore, formatter) : myTypedDispatchTable[tag]; - (myTypedDispatchTable[tag] as TypedPubItemHandler).addSubscription(handler); - } - - public void subscribe(string tag, RawDataHandler handler) { - myRawDispatchTable[tag] = !myRawDispatchTable.Contains(tag) ? - new RawPubItemHandler(tag, mySubsCore) : myRawDispatchTable[tag]; - (myRawDispatchTable[tag] as RawPubItemHandler).addSubscription(handler); - } - - public void unsubscribe(string tag, Handler handler) { - if (myTypedDispatchTable.Contains(tag)) { - (myTypedDispatchTable[tag] as TypedPubItemHandler).removeSubscription(handler); - if ((myTypedDispatchTable[tag] as SingleSubHandler).nSubscribers == 0) - myTypedDispatchTable.Remove(tag); - } else { } - // BlueEventLog.logEvent("Trying to unsubscribe a PubItem tag that was never subsribed", BlueEventLevel.Error); - } - - public bool alreadySubscribed(string tag) { - return myTypedDispatchTable.Contains(tag) || myRawDispatchTable.Contains(tag); - } - - public void unsubscribe(string tag, RawDataHandler handler) { - if (myRawDispatchTable.Contains(tag)) { - (myRawDispatchTable[tag] as RawPubItemHandler).removeSubscription(handler); - if ((myRawDispatchTable[tag] as SingleSubHandler).nSubscribers == 0) - myRawDispatchTable.Remove(tag); - } else { } - // BlueEventLog.logEvent ("Trying to unsubscribe a Raw tag that was never subsribed", BlueEventLevel.Warning); - } - - private void myFireEvents(Dictionary> update_map) { + private void _fire_events(IEnumerable updates) { // TODO - bts.Log.assert(false); - // foreach (DictionaryEntry de in updates) { - // Stream s = StreamUtilityMethods.getAggregatedStream((de.Value as ArrayList).ToArray(typeof(Stream)) as Stream[]); - // if (myTypedDispatchTable.Contains(de.Key)) - // (myTypedDispatchTable[de.Key] as TypedPubItemHandler).update(de.Key.ToString(), s); - // else if (myRawDispatchTable.Contains(de.Key)) - // (myRawDispatchTable[de.Key] as RawPubItemHandler).update(de.Key.ToString(), s); - // else { } - // // BlueEventLog.logEvent ("Received update for an unsubscribed tag", BlueEventLevel.Warning); - // } + foreach (var u in updates) { + List cbs = null; + if (_dispatch_table.TryGetValue(u.tag, out cbs)) { + foreach (var cb in cbs) { + cb.Invoke(u.tag, u.data); + } + } + } } public IPubSubRouter router { get { - return mySubsCore.get_router(); + return _subcore.get_router(); } } public static ManagedSubscriber instance { - get { return mySelf = mySelf == null ? new ManagedSubscriber() : mySelf; } + get { return _self = _self == null ? new ManagedSubscriber() : _self; } } #region IDisposable Members public void Dispose() { - mySubsCore.Dispose(); + _subcore.Dispose(); } #endregion diff --git a/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/RawPubItemHandler.cs b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/RawPubItemHandler.cs index e8e7a1fe3..fc141b62e 100644 --- a/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/RawPubItemHandler.cs +++ b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/RawPubItemHandler.cs @@ -1,23 +1,24 @@ -using System; -using System.IO; +// using System; +// using System.IO; -namespace BCG -{ - internal class RawPubItemHandler : SingleSubHandler - { - public RawPubItemHandler (string tag, SubscriberCore subCore) : - base (tag, subCore) - {} +//TODO:remove +// namespace BCG +// { +// internal class RawPubItemHandler : SingleSubHandler +// { +// public RawPubItemHandler (string tag, SubscriberCore subCore) : +// base (tag, subCore) +// {} - protected override void myFire (Delegate del) - { - (del as RawDataHandler)(myTag, myData as Stream); - } +// protected override void myFire (Delegate del) +// { +// (del as RawDataHandler)(myTag, myData as Stream); +// } - public override void update (string tag, Stream s) - { - myData = s; - myFire (myDelegate); - } - } -} +// public override void update (string tag, Stream s) +// { +// myData = s; +// myFire (myDelegate); +// } +// } +// } diff --git a/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/SingleSubHandler.cs b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/SingleSubHandler.cs index 5cb23505f..6288aa273 100644 --- a/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/SingleSubHandler.cs +++ b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/SingleSubHandler.cs @@ -1,52 +1,53 @@ -using System; -using System.IO; +// using System; +// using System.IO; -namespace BCG -{ - abstract internal class SingleSubHandler - { - protected SubscriberCore mySubCore; - protected Delegate myDelegate; - protected string myTag; - protected object myData; +// TODO: remove +// namespace BCG +// { +// abstract internal class SingleSubHandler +// { +// protected SubscriberCore mySubCore; +// protected Delegate myDelegate; +// protected string myTag; +// protected object myData; - public SingleSubHandler (string tag, SubscriberCore subCore) - { - myTag = tag; - mySubCore = subCore; - } +// public SingleSubHandler (string tag, SubscriberCore subCore) +// { +// myTag = tag; +// mySubCore = subCore; +// } - public virtual void addSubscription (Delegate del) - { - if (myDelegate == null) - mySubCore.subscribe (myTag); - myDelegate = Delegate.Combine (myDelegate, del); - if (myData != null) - myFire (del); - } +// public virtual void addSubscription (Delegate del) +// { +// if (myDelegate == null) +// mySubCore.subscribe (myTag); +// myDelegate = Delegate.Combine (myDelegate, del); +// if (myData != null) +// myFire (del); +// } - public virtual void removeSubscription (Delegate del) - { - if (myDelegate != null) - { - myDelegate = Delegate.Remove (myDelegate, del); - if (myDelegate == null) - { - mySubCore.unsubscribe (myTag); - myData = null; - } - } - } +// public virtual void removeSubscription (Delegate del) +// { +// if (myDelegate != null) +// { +// myDelegate = Delegate.Remove (myDelegate, del); +// if (myDelegate == null) +// { +// mySubCore.unsubscribe (myTag); +// myData = null; +// } +// } +// } - public int nSubscribers - { - get { return myDelegate == null ? 0 : myDelegate.GetInvocationList().Length; } - } +// public int nSubscribers +// { +// get { return myDelegate == null ? 0 : myDelegate.GetInvocationList().Length; } +// } - public abstract void update (string tag, Stream s); +// public abstract void update (string tag, Stream s); - protected abstract void myFire (Delegate del); +// protected abstract void myFire (Delegate del); - } +// } -} +// } diff --git a/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/TypedPubItemHandler.cs b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/TypedPubItemHandler.cs index 2a3167686..db67b2caf 100644 --- a/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/TypedPubItemHandler.cs +++ b/src/DataStreamSuite/PubSub/Client/SubscriptionUtilities/TypedPubItemHandler.cs @@ -1,37 +1,37 @@ -using System; -using System.IO; -using System.Runtime.Serialization.Formatters.Binary; +// using System; +// using System.IO; +// using System.Runtime.Serialization.Formatters.Binary; +// TODO: remove +// namespace BCG +// { +// internal class TypedPubItemHandler : SingleSubHandler +// { +// private IGenericFormatter myFormatter; -namespace BCG -{ - internal class TypedPubItemHandler : SingleSubHandler - { - private IGenericFormatter myFormatter; +// public TypedPubItemHandler (string tag, SubscriberCore subCore) +// : this (tag, subCore, new GenericBinaryFormatter ()) +// { +// } - public TypedPubItemHandler (string tag, SubscriberCore subCore) - : this (tag, subCore, new GenericBinaryFormatter ()) - { - } +// public TypedPubItemHandler (string tag, SubscriberCore subCore, IGenericFormatter formatter) +// : base (tag, subCore) +// { +// myFormatter = formatter; +// } - public TypedPubItemHandler (string tag, SubscriberCore subCore, IGenericFormatter formatter) - : base (tag, subCore) - { - myFormatter = formatter; - } +// protected override void myFire (Delegate del) +// { +// (del as Handler)(myData as IPubItem); +// } - protected override void myFire (Delegate del) - { - (del as Handler)(myData as IPubItem); - } - - public override void update (string tag, Stream s) - { - while (s.Position < s.Length) - { - myData = myFormatter.Deserialize (s); - myFire (myDelegate); - } - } - } -} +// public override void update (string tag, Stream s) +// { +// while (s.Position < s.Length) +// { +// myData = myFormatter.Deserialize (s); +// myFire (myDelegate); +// } +// } +// } +// } diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs index 57ea31643..48bc7db0b 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs @@ -39,7 +39,9 @@ namespace bts { } private void _on_accept(IAsyncResult res) { - bts.Log.Gen.info("AsyncTCPConnectionAcceptor_APM._on_accept state: {res.AsyncState}"); + var log_msg = $"AsyncTCPConnectionAcceptor_APM._on_accept state: {res.AsyncState}"; + Log.Gen.info(log_msg); + Console.WriteLine(log_msg); string key = res.AsyncState.ToString(); if (_key_map.Contains(key)) { return; diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs index 763470931..242f9a02d 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs @@ -1,586 +1,586 @@ -//This implementation uses APM the beginFunc, endFunc pattern -// https://learn.microsoft.com/en-us/dotnet/standard/asynchronous-programming-patterns/asynchronous-programming-model-apm#beginning-an-asynchronous-operation - -/// -/// This class, mantains a connection to a peer socket and using the APM -/// model to communicate. It sends heartbeats at regular -/// intervals (config). It is in active state when messages -/// are sent and received without an error state. It is assumed -/// dead state when either of the following happens -/// BeginReceive returns a SocketError -/// BeginSend returns a SocketError -/// EndReceive returns a SocketError -/// EndSend returns a SocketError -/// As soon as it assumes dead state it stops sending or -/// receiving any messages (including heartbeats). -/// The user of this class can either detect that by -/// missing heartbeats or checking the isConnected property. -/// If the peer dies in a 'silent' way, where none of the above -/// methods indicate an error, missing heartbeats is guaranteed -/// to detect it. -/// - -using System; -using System.Collections.Generic; -using System.Text; -using System.Net; -using System.Net.Sockets; -using System.IO; -using BCG.Utilities.General; -using System.Runtime.InteropServices; -using BCG; -using System.Collections.ObjectModel; -using System.Numerics; -using System.Reflection.Metadata; -using System.Threading.Tasks; -using System.Linq; -using Bts.PubSub; -using Spark.Core; - -namespace bts { - public class AsyncTCPConnection_APM : ITCPConnection, IWorker, IDisposable { - - // serves as the base class for both Send and ReeceiveMessage classes. - // since both TCP sends and receives happen in multiple transfers, this - // provides a wrapper around copying the partial transfers into the complete buffer - public class AsyncMessageBase { - readonly string msg_id = Guid.NewGuid().ToString(); - // every BeginSend or BeginReceive has a unique key associated with it - public int key; - // total msg that needs to be transfered - public byte[] buffer; - // as data is transferred in chunks, transfer_ptr moves ahead by amount of data transferred. - // everything < transfer_ptr has been successfully transferred, - // ie this is pointing to where the next transfer will start from - public int transfer_ptr; - protected int _n_transfers = 0; - public AsyncMessageBase(byte[] the_buffer, int the_offset) { - buffer = the_buffer; - transfer_ptr = the_offset; - } - public int remaining_size { - get { - return buffer.Length - transfer_ptr; - } - } - - public int original_size => buffer.Length; - - public void advance_transfer_pointer(int advance_by) { - transfer_ptr += advance_by; - _n_transfers++; - if (transfer_ptr > original_size) { - bts.Log.Gen.error($"transfer went beyond initial size {ToString()}"); - } - // TODO: log if it goes past the buffer - } - - public bool fully_transfered() { - return transfer_ptr >= original_size; - } - - public override string ToString() { - return $"{GetType().ToString()} msg_id: {msg_id} key: {key} original_length: {original_size} transfer_ptr: {transfer_ptr} n_transfers: {_n_transfers} " + - $"fully_transfered: {fully_transfered()}"; - } - } - public class AsyncSendMessage : AsyncMessageBase { - public string message_id; - public SendConfirmationDelegate on_send_del; - //private int _n_transfers = 0; - - public AsyncSendMessage(byte[] the_buffer, int the_offset) - : this(the_buffer, the_offset, String.Empty, null) { } - - public AsyncSendMessage(byte[] the_buffer, int the_transfer_ptr, string the_message_id, SendConfirmationDelegate the_on_send_del) - : base(the_buffer, the_transfer_ptr) { - message_id = the_message_id; - on_send_del = the_on_send_del; - } - public override string ToString() { - return base.ToString(); - } - - } - - public class AsyncReceiveMessage : AsyncMessageBase { - public enum ReadState { - reading_size, - reading_data - }; - readonly ReadState read_state; - - public AsyncReceiveMessage(int size, ReadState the_read_state) : - this(new byte[size], 0, the_read_state) { } - public AsyncReceiveMessage(byte[] the_buffer, int the_transfer_ptr, ReadState the_read_state) - : base(the_buffer, the_transfer_ptr) { - read_state = the_read_state; - } - - public int read_int32(int start_idx = 0) { - return BitConverter.ToInt32(buffer, start_idx); - } - - public byte[] read_bytes(int start_idx) { - byte[] rv = new byte[buffer.Length - start_idx]; - Array.Copy(buffer, start_idx, rv, 0, rv.Length); - return rv; - } - - public bool is_reading_size() => read_state == ReadState.reading_size; - } - - public class ErrorMessage { - public readonly SocketError? socket_error = null; - public readonly Exception exception = null; - public readonly bool is_exception = false; - public readonly string context = String.Empty; - public ErrorMessage(SocketError socket_error_, string context_) { - socket_error = socket_error_; - context = context_; - } - public ErrorMessage(Exception exception_, string context_) { - is_exception = true; - exception = exception_; - context = context_; - } - - public bool oom => exception != null && exception is OutOfMemoryException; - - public override string ToString() { - if (is_exception) - return $"ErrorMessage exception: {exception} context: {context}"; - return $"socket_error: {socket_error} context: {context}"; - } - - - } - - #region work around for .Net socket's LocalEndPoint bug - - // [StructLayout(LayoutKind.Sequential)] - // private struct in_addr { - // [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)] - // public byte[] sin_addr; - // } - - // [StructLayout(LayoutKind.Sequential)] - // private struct sockaddr { - // public short sin_family; - // public ushort sin_port; - // public in_addr sin_addr; - // [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)] - // public byte[] sin_zero; - // } - - // [DllImport("wsock32.dll")] - // private static extern int getsockname(IntPtr s, ref sockaddr name, ref int namelen); - - // [DllImport("wsock32.dll")] - // private static extern int getpeername(IntPtr s, ref sockaddr name, ref int namelen); - - #endregion - - private const int DATA_MSG_SIGNAL = 0; - private const int HB_MSG_SIGNAL = 1; - private int _send_key = 0, _receive_key = 0; - private HashSet _read_key_map = new HashSet(), - _send_key_map = new HashSet(); - private readonly int _max_msg_size = PubSubConfig.Instance.MaxPacketSize.assert_value(); - private readonly RemoteConnectionInfo _peer_conn_info, _self_conn_info; - private readonly ITCPConnection.ConnectionSide _connection_side; - private readonly Socket _sock; - private ITCPMessageHandler _message_handler = null; - - private bool _is_alive = true; - - private byte[] _cached_hb_message = null; - private readonly TimeSpan _hb_period = TimeSpan.FromSeconds(1); // TODO get it from config - private MasterWorker _master_worker = MasterWorker.getInstance(true); - private bool _hb_received_yet = false, _data_received_yet = false, _hb_sent_once = false; - private DateTime _last_hb_received_time = new DateTime(1900, 1, 1); - - public AsyncTCPConnection_APM(Socket sock, TimeSpan hb_period, ITCPConnection.ConnectionSide connection_side) { - _sock = sock; - _hb_period = hb_period; - _connection_side = connection_side; - _sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontRoute, true); - string ip; - int port; - getPeerAddress(out ip, out port); - _peer_conn_info = new RemoteConnectionInfo(ip, port); - getSelfAddress(out ip, out port); - _self_conn_info = new RemoteConnectionInfo(ip, port); - bts.Log.Gen.info($"AsyncTCPConnection_APM created {ToString()}"); - } - - public override string ToString() { - return $"AsyncTCPConnection_APM side: {_connection_side} peer: {_peer_conn_info}, self: {_self_conn_info}"; - } - - #region ITCPConnection Members - - public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del) { - conf_id = Guid.NewGuid().ToString(); - byte[] msg = _build_data_message(data); - _begin_or_resume_sending(new AsyncSendMessage(msg, 0, conf_id, del)); - } - - public void send(byte[] data) { - byte[] msg = _build_data_message(data); - _begin_or_resume_sending(new AsyncSendMessage(msg, 0)); - } - - public void register(ITCPMessageHandler messageHandler) { - bts.Log.assert_d(_message_handler == null, "Message Handler is already assigned"); - bts.Log.Gen.info($"AsyncTCPConnection_APM for peer = {_peer_conn_info} being registered"); - _message_handler = messageHandler; - _master_worker.addWorker(this, _hb_period); - // we send only when send is called but read continoulsy, starting here - _read_next_msg(); - } - - public void getPeerAddress(out string ip, out int port) { - ip = ""; - port = 0; - if (_is_alive) { - IPEndPoint ipEp = _sock.RemoteEndPoint as IPEndPoint; - port = ipEp.Port; - ip = ipEp.Address.ToString(); - } - } - - public void getSelfAddress(out string ip, out int port) { - ip = ""; - port = 0; - if (_is_alive) { - IPEndPoint ipEp = _sock.LocalEndPoint as IPEndPoint; - ip = ipEp.Address.ToString(); - port = ipEp.Port; - } - } - - public bool isDisposed { - get { return !_is_alive; } - } - - #endregion - - #region IWorker Members - - bool _last_connected = false; - public void doWork() { - if (!_is_alive) - return; - return; - // disable hbs for now - if (!_hb_sent_once) { - bts.Log.Gen.info("Sending heartbeat to peer {_peer_conn_info} for the first time"); - _hb_sent_once = true; - } - var msg = _build_hb_message(); - _begin_or_resume_sending(new AsyncSendMessage(msg, 0)); - } - - private static int _tid() { - var tid = System.Threading.Thread.CurrentThread.ManagedThreadId; - return tid; - } - - private static string _cb_txt(IAsyncResult res) { - return $"completed?: {res.IsCompleted} sync?: {res.CompletedSynchronously}"; - } - - #endregion - - #region sending data - - private void _begin_or_resume_sending(AsyncSendMessage message) { - bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_sending this: {ToString()} tid: {_tid()} message: {message} _is_alive: {_is_alive}"); - if (_is_alive) { - try { - SocketError begin_send_err; - // since we get message on on_send we dont need to keep the state around - message.key = _send_key++; - var res = _sock.BeginSend(message.buffer, message.transfer_ptr, message.remaining_size, SocketFlags.None, - out begin_send_err, new AsyncCallback(_on_send), message); - if (begin_send_err != SocketError.Success) - _handle_error(new ErrorMessage(begin_send_err, "_begin_or_resume_sending")); - else { - // successfully sent, all we need to do here is make sure if it was sent asynchronously - // if not call _on_send explicitely - if (res != null && res.IsCompleted && res.CompletedSynchronously) { - // for small messages on LANs this can happen, we will on_send to deal with it the same way - Task.Run(() => { - bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_sending calling _on_send explicitely"); - _on_send(res); - }); - } - } - } catch (Exception ex) { - _handle_error(new ErrorMessage(ex, "_begin_or_resume_sending")); - } - } - } - - private void _on_send(IAsyncResult res) { - AsyncSendMessage message = res.AsyncState as AsyncSendMessage; - if (_send_key_map.Contains(message.key)) { - bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_send key: {message.key} already handled"); - return; - } - _send_key_map.Add(message.key); - bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: pre endSend this: {ToString()} tid: {_tid()} " + - $"res: {_cb_txt(res)} message: {res.AsyncState} _is_alive: {_is_alive}"); - if (_is_alive) { - try { - SocketError send_err; - int bytes_sent = _sock.EndSend(res, out send_err); - //message.offset += bytesSent; - message.advance_transfer_pointer(bytes_sent); - if (bytes_sent == 0) { - bts.Log.Gen.error($"AsyncTCPConnection_APM._on_send this: {this} sent 0 bytes, throwing exception"); - _handle_error(new ErrorMessage(new ApplicationException("Sent 0 bytes, peer socket is likely shutdown"), - "_on_send")); - return; - } - bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: post endSend this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} " + - $"message: {res.AsyncState} bytes_sent: {bytes_sent} _is_alive: {_is_alive}"); - if (send_err == SocketError.Success) { - // successful send of some bytes - //if (message.offset == message.buffer.Length) { - if (message.fully_transfered()) { - // fully sent, if confirmation needs to be sent out - if (!String.IsNullOrEmpty(message.message_id)) { - message.on_send_del(message.message_id, true); - } - } else { - // there is more to send - _begin_or_resume_sending(message); - } - } else { - _handle_error(new ErrorMessage(send_err, "_on_send")); - } - } catch (Exception ex) { - _handle_error(new ErrorMessage(ex, "_on_send")); - } - } - } - - /// - /// Assembles the heartbeat message, and uses it for - /// future references - /// - /// - private byte[] _build_hb_message() { - if (_cached_hb_message == null) { - BinaryWriter bw = new BinaryWriter(new MemoryStream()); - // the total msg =[msg_sz, msg_type] - int msg_sz = 4; - bw.Write(msg_sz); - bw.Write(HB_MSG_SIGNAL); - _cached_hb_message = (bw.BaseStream as MemoryStream).ToArray(); - } - return _cached_hb_message; - } - - // writes size prefixed msg - private byte[] _build_data_message(byte[] data) { - // TODO, avoid unecessary memory creation (write directly into byte[] using BitConverter) - BinaryWriter bw = new BinaryWriter(new MemoryStream()); - // the total msg =[msg_sz, msg_type, data] - var msg_sz = data.Length + 4; - bw.Write(msg_sz); // +4 for the type of msg - // to indicate data vs hb - bw.Write(DATA_MSG_SIGNAL); // for data - // if we want to id every message, this is the place to do it - bw.Write(data); - return (bw.BaseStream as MemoryStream).ToArray(); - } - - #endregion - - #region receiving data - - private void _read_next_msg() { - _read_size(); - } - - private void _read_size() { - _begin_or_resume_reading(new AsyncReceiveMessage(4, AsyncReceiveMessage.ReadState.reading_size)); - } - - private void _read_data(int data_size) { - // TODO, this could be a problem is the size is too big - maybe read it in chunks (later) - _begin_or_resume_reading(new AsyncReceiveMessage(data_size, AsyncReceiveMessage.ReadState.reading_data)); - } - - private void _begin_or_resume_reading(AsyncReceiveMessage rcv_msg) { - bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_reading: this: {ToString()} tid: {_tid()} message: {rcv_msg} _is_alive: {_is_alive}"); - if (!_is_alive) - return; - try { - SocketError read_sock_err; - rcv_msg.key = _receive_key++; - var res = _sock.BeginReceive(rcv_msg.buffer, rcv_msg.transfer_ptr, - rcv_msg.remaining_size, - SocketFlags.None, out read_sock_err, - new AsyncCallback(_on_read), - rcv_msg); - if (read_sock_err != SocketError.Success) { - bts.Log.Gen.error($"_begin_or_resume_reading this: {this} error: {read_sock_err} rcv_msg: {rcv_msg}"); - _handle_error(new ErrorMessage(read_sock_err, "_beging_or_resume_reading")); - } else { - // successfully started receiving - if (res != null && res.IsCompleted && res.CompletedSynchronously) { - // make it follow the same code path - Task.Run(() => { - bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_reading calling _on_send explicitely"); - _on_read(res); - } - ); - } - } - } catch (Exception ex) { - _handle_error(new ErrorMessage(ex, "_beging_or_resume_reading")); - } - } - - private void _on_read(IAsyncResult res) { - AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage; - if (_read_key_map.Contains(rcv_message.key)) { - bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_read key: {rcv_message.key} already handled"); - return; - } - _read_key_map.Add(rcv_message.key); - if (!_is_alive) - return; - try { - SocketError read_err; - int bytes_read = _sock.EndReceive(res, out read_err); - rcv_message.advance_transfer_pointer(bytes_read); - bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read: this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} bytes_read: {bytes_read} message: {res.AsyncState} _is_alive: {_is_alive} read_err: {read_err}"); - if (bytes_read == 0) { - bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, throwing exception"); - _handle_error(new ErrorMessage(new ApplicationException("Read 0 bytes, peer socket is likely shutdown"), "_on_read")); - return; - } - //return; - if (read_err == SocketError.Success) { - if (rcv_message.fully_transfered()) { - if (rcv_message.is_reading_size()) { - var msg_size = _on_read_size_completely(rcv_message); - if (msg_size != null && msg_size.HasValue) { - _read_data(msg_size.Value); - } - return; - } else { - // data is read - int msg_type = rcv_message.read_int32(); - bts.Log.assert(msg_type == HB_MSG_SIGNAL || msg_type == DATA_MSG_SIGNAL); - if (msg_type == DATA_MSG_SIGNAL) { - _on_read_data_completely(rcv_message); - _read_next_msg(); - return; - return; - } else if (msg_type == HB_MSG_SIGNAL) { - _on_read_hb_completely(rcv_message); - _read_next_msg(); - return; - } - // go back to reading size for a new message - //_read_size(); - } - } else { - // msg has not been fully rd, we have already advanced the transfer ptr - _begin_or_resume_reading(rcv_msg: rcv_message); - } - } else { - _handle_error(new ErrorMessage(read_err, "_on_read")); - } - } catch (Exception ex) { - _handle_error(new ErrorMessage(ex, "_on_read")); - } - } - - #endregion - - private int? _on_read_size_completely(AsyncReceiveMessage msg) { - bts.Log.assert(msg.is_reading_size()); - bts.Log.assert(msg.fully_transfered()); - bts.Log.assert(msg.original_size == 4); - var msg_sz = msg.read_int32(); - bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely this: {this} msg: {msg} msg_sz: {msg_sz}"); - if (msg_sz > _max_msg_size) { - string err_msg = $"Encountered malformed message, prefixed_size: {msg_sz}"; - _handle_error(new ErrorMessage(new ApplicationException(err_msg), "_on_read")); - return null; - } else { - // sz is determined, now we read that many bytes - //_read_data(msg_sz); - return msg_sz; - } - } - - private void _on_read_hb_completely(AsyncReceiveMessage msg) { - bts.Log.assert(!msg.is_reading_size()); - bts.Log.assert(msg.fully_transfered()); - bts.Log.assert(msg.original_size == 4); - bts.Log.assert(msg.read_int32(0) == HB_MSG_SIGNAL); - if (!_hb_received_yet) { - _hb_received_yet = true; - bts.Log.Gen.info($"AsyncTCPConnection_APM HB received for the first time from peer {_peer_conn_info}"); - } - _last_hb_received_time = DateTime.Now; - _message_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_conn_info)); - } - private void _on_read_data_completely(AsyncReceiveMessage msg) { - bts.Log.assert(!msg.is_reading_size()); - bts.Log.assert(msg.fully_transfered()); - var type = msg.read_int32(0); - bts.Log.assert(type == DATA_MSG_SIGNAL); - bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_data_completely this: {this} msg: {msg} type: {type}"); - byte[] data = msg.read_bytes(4); - ///callbacks can be made through thread pool to avoid this - ///thread from being blocked, by this callback - if (!_data_received_yet) { - _data_received_yet = true; - bts.Log.Gen.info($"AsyncTCPConnection_APM data received for the first time from peer {_peer_conn_info}"); - } - // could be an index here - _message_handler.onData(new RemoteMessage(data, RemoteMessageType.Data, _peer_conn_info)); - } - - #region IDisposable Members - - public void Dispose() { - // socket is closed and disposed here - lock (this) { - if (_is_alive) { - _master_worker.removeWorker(this); - _is_alive = false; - bts.Log.Gen.info($"AsycnTCPConnection.Dispose this: {this} being disposed, last HB was received at {_last_hb_received_time}"); - try { - _sock.Shutdown(SocketShutdown.Both); - } catch (Exception ex) { - String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket"); - if (ex is OutOfMemoryException) - throw ex; - } - _message_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _peer_conn_info)); - } - } - } - - #endregion - - private void _handle_error(ErrorMessage err_msg) { - lock (this) { - Log.Gen.info($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}"); - if (err_msg.oom) - throw err_msg.exception; - Dispose(); - } - } - } -} +//This implementation uses APM the beginFunc, endFunc pattern +// https://learn.microsoft.com/en-us/dotnet/standard/asynchronous-programming-patterns/asynchronous-programming-model-apm#beginning-an-asynchronous-operation + +/// +/// This class, mantains a connection to a peer socket and using the APM +/// model to communicate. It sends heartbeats at regular +/// intervals (config). It is in active state when messages +/// are sent and received without an error state. It is assumed +/// dead state when either of the following happens +/// BeginReceive returns a SocketError +/// BeginSend returns a SocketError +/// EndReceive returns a SocketError +/// EndSend returns a SocketError +/// As soon as it assumes dead state it stops sending or +/// receiving any messages (including heartbeats). +/// The user of this class can either detect that by +/// missing heartbeats or checking the isConnected property. +/// If the peer dies in a 'silent' way, where none of the above +/// methods indicate an error, missing heartbeats is guaranteed +/// to detect it. +/// + +using System; +using System.Collections.Generic; +using System.Text; +using System.Net; +using System.Net.Sockets; +using System.IO; +using BCG.Utilities.General; +using System.Runtime.InteropServices; +using BCG; +using System.Collections.ObjectModel; +using System.Numerics; +using System.Reflection.Metadata; +using System.Threading.Tasks; +using System.Linq; +using Bts.PubSub; +using Spark.Core; + +namespace bts { + public class AsyncTCPConnection_APM : ITCPConnection, IWorker, IDisposable { + + // serves as the base class for both Send and ReeceiveMessage classes. + // since both TCP sends and receives happen in multiple transfers, this + // provides a wrapper around copying the partial transfers into the complete buffer + public class AsyncMessageBase { + readonly string msg_id = Guid.NewGuid().ToString(); + // every BeginSend or BeginReceive has a unique key associated with it + public int key; + // total msg that needs to be transfered + public byte[] buffer; + // as data is transferred in chunks, transfer_ptr moves ahead by amount of data transferred. + // everything < transfer_ptr has been successfully transferred, + // ie this is pointing to where the next transfer will start from + public int transfer_ptr; + protected int _n_transfers = 0; + public AsyncMessageBase(byte[] the_buffer, int the_offset) { + buffer = the_buffer; + transfer_ptr = the_offset; + } + public int remaining_size { + get { + return buffer.Length - transfer_ptr; + } + } + + public int original_size => buffer.Length; + + public void advance_transfer_pointer(int advance_by) { + transfer_ptr += advance_by; + _n_transfers++; + if (transfer_ptr > original_size) { + bts.Log.Gen.error($"transfer went beyond initial size {ToString()}"); + } + // TODO: log if it goes past the buffer + } + + public bool fully_transfered() { + return transfer_ptr >= original_size; + } + + public override string ToString() { + return $"{GetType().ToString()} msg_id: {msg_id} key: {key} original_length: {original_size} transfer_ptr: {transfer_ptr} n_transfers: {_n_transfers} " + + $"fully_transfered: {fully_transfered()}"; + } + } + public class AsyncSendMessage : AsyncMessageBase { + public string message_id; + public SendConfirmationDelegate on_send_del; + //private int _n_transfers = 0; + + public AsyncSendMessage(byte[] the_buffer, int the_offset) + : this(the_buffer, the_offset, String.Empty, null) { } + + public AsyncSendMessage(byte[] the_buffer, int the_transfer_ptr, string the_message_id, SendConfirmationDelegate the_on_send_del) + : base(the_buffer, the_transfer_ptr) { + message_id = the_message_id; + on_send_del = the_on_send_del; + } + public override string ToString() { + return base.ToString(); + } + + } + + public class AsyncReceiveMessage : AsyncMessageBase { + public enum ReadState { + reading_size, + reading_data + }; + readonly ReadState read_state; + + public AsyncReceiveMessage(int size, ReadState the_read_state) : + this(new byte[size], 0, the_read_state) { } + public AsyncReceiveMessage(byte[] the_buffer, int the_transfer_ptr, ReadState the_read_state) + : base(the_buffer, the_transfer_ptr) { + read_state = the_read_state; + } + + public int read_int32(int start_idx = 0) { + return BitConverter.ToInt32(buffer, start_idx); + } + + public byte[] read_bytes(int start_idx) { + byte[] rv = new byte[buffer.Length - start_idx]; + Array.Copy(buffer, start_idx, rv, 0, rv.Length); + return rv; + } + + public bool is_reading_size() => read_state == ReadState.reading_size; + } + + public class ErrorMessage { + public readonly SocketError? socket_error = null; + public readonly Exception exception = null; + public readonly bool is_exception = false; + public readonly string context = String.Empty; + public ErrorMessage(SocketError socket_error_, string context_) { + socket_error = socket_error_; + context = context_; + } + public ErrorMessage(Exception exception_, string context_) { + is_exception = true; + exception = exception_; + context = context_; + } + + public bool oom => exception != null && exception is OutOfMemoryException; + + public override string ToString() { + if (is_exception) + return $"ErrorMessage exception: {exception} context: {context}"; + return $"socket_error: {socket_error} context: {context}"; + } + + + } + + #region work around for .Net socket's LocalEndPoint bug + + // [StructLayout(LayoutKind.Sequential)] + // private struct in_addr { + // [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)] + // public byte[] sin_addr; + // } + + // [StructLayout(LayoutKind.Sequential)] + // private struct sockaddr { + // public short sin_family; + // public ushort sin_port; + // public in_addr sin_addr; + // [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)] + // public byte[] sin_zero; + // } + + // [DllImport("wsock32.dll")] + // private static extern int getsockname(IntPtr s, ref sockaddr name, ref int namelen); + + // [DllImport("wsock32.dll")] + // private static extern int getpeername(IntPtr s, ref sockaddr name, ref int namelen); + + #endregion + + private const int DATA_MSG_SIGNAL = 0; + private const int HB_MSG_SIGNAL = 1; + private int _send_key = 0, _receive_key = 0; + private HashSet _read_key_map = new HashSet(), + _send_key_map = new HashSet(); + private readonly int _max_msg_size = PubSubConfig.Instance.MaxPacketSize.assert_value(); + private readonly RemoteConnectionInfo _peer_conn_info, _self_conn_info; + private readonly ITCPConnection.ConnectionSide _connection_side; + private readonly Socket _sock; + private ITCPMessageHandler _message_handler = null; + + private bool _is_alive = true; + + private byte[] _cached_hb_message = null; + private readonly TimeSpan _hb_period = TimeSpan.FromSeconds(1); // TODO get it from config + private MasterWorker _master_worker = MasterWorker.getInstance(true); + private bool _hb_received_yet = false, _data_received_yet = false, _hb_sent_once = false; + private DateTime _last_hb_received_time = new DateTime(1900, 1, 1); + + public AsyncTCPConnection_APM(Socket sock, TimeSpan hb_period, ITCPConnection.ConnectionSide connection_side) { + _sock = sock; + _hb_period = hb_period; + _connection_side = connection_side; + _sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontRoute, true); + string ip; + int port; + getPeerAddress(out ip, out port); + _peer_conn_info = new RemoteConnectionInfo(ip, port); + getSelfAddress(out ip, out port); + _self_conn_info = new RemoteConnectionInfo(ip, port); + bts.Log.Gen.info($"AsyncTCPConnection_APM created {ToString()}"); + } + + public override string ToString() { + return $"AsyncTCPConnection_APM side: {_connection_side} peer: {_peer_conn_info}, self: {_self_conn_info}"; + } + + #region ITCPConnection Members + + public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del) { + conf_id = Guid.NewGuid().ToString(); + byte[] msg = _build_data_message(data); + _begin_or_resume_sending(new AsyncSendMessage(msg, 0, conf_id, del)); + } + + public void send(byte[] data) { + byte[] msg = _build_data_message(data); + _begin_or_resume_sending(new AsyncSendMessage(msg, 0)); + } + + public void register(ITCPMessageHandler messageHandler) { + bts.Log.assert_d(_message_handler == null, "Message Handler is already assigned"); + //bts.Log.Gen.info($"AsyncTCPConnection_APM for peer = {_peer_conn_info} being registered"); + _message_handler = messageHandler; + _master_worker.addWorker(this, _hb_period); + // we send only when send is called but read continoulsy, starting here + _read_next_msg(); + } + + public void getPeerAddress(out string ip, out int port) { + ip = ""; + port = 0; + if (_is_alive) { + IPEndPoint ipEp = _sock.RemoteEndPoint as IPEndPoint; + port = ipEp.Port; + ip = ipEp.Address.ToString(); + } + } + + public void getSelfAddress(out string ip, out int port) { + ip = ""; + port = 0; + if (_is_alive) { + IPEndPoint ipEp = _sock.LocalEndPoint as IPEndPoint; + ip = ipEp.Address.ToString(); + port = ipEp.Port; + } + } + + public bool isDisposed { + get { return !_is_alive; } + } + + #endregion + + #region IWorker Members + + bool _last_connected = false; + public void doWork() { + if (!_is_alive) + return; + return; + // disable hbs for now + if (!_hb_sent_once) { + bts.Log.Gen.info("Sending heartbeat to peer {_peer_conn_info} for the first time"); + _hb_sent_once = true; + } + var msg = _build_hb_message(); + _begin_or_resume_sending(new AsyncSendMessage(msg, 0)); + } + + private static int _tid() { + var tid = System.Threading.Thread.CurrentThread.ManagedThreadId; + return tid; + } + + private static string _cb_txt(IAsyncResult res) { + return $"completed?: {res.IsCompleted} sync?: {res.CompletedSynchronously}"; + } + + #endregion + + #region sending data + + private void _begin_or_resume_sending(AsyncSendMessage message) { + //bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_sending this: {ToString()} tid: {_tid()} message: {message} _is_alive: {_is_alive}"); + if (_is_alive) { + try { + SocketError begin_send_err; + // since we get message on on_send we dont need to keep the state around + message.key = _send_key++; + var res = _sock.BeginSend(message.buffer, message.transfer_ptr, message.remaining_size, SocketFlags.None, + out begin_send_err, new AsyncCallback(_on_send), message); + if (begin_send_err != SocketError.Success) + _handle_error(new ErrorMessage(begin_send_err, "_begin_or_resume_sending")); + else { + // successfully sent, all we need to do here is make sure if it was sent asynchronously + // if not call _on_send explicitely + if (res != null && res.IsCompleted && res.CompletedSynchronously) { + // for small messages on LANs this can happen, we will on_send to deal with it the same way + Task.Run(() => { + // bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_sending calling _on_send explicitely"); + _on_send(res); + }); + } + } + } catch (Exception ex) { + _handle_error(new ErrorMessage(ex, "_begin_or_resume_sending")); + } + } + } + + private void _on_send(IAsyncResult res) { + AsyncSendMessage message = res.AsyncState as AsyncSendMessage; + if (_send_key_map.Contains(message.key)) { + bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_send key: {message.key} already handled"); + return; + } + _send_key_map.Add(message.key); + // bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: pre endSend this: {ToString()} tid: {_tid()} " + + // $"res: {_cb_txt(res)} message: {res.AsyncState} _is_alive: {_is_alive}"); + if (_is_alive) { + try { + SocketError send_err; + int bytes_sent = _sock.EndSend(res, out send_err); + //message.offset += bytesSent; + message.advance_transfer_pointer(bytes_sent); + if (bytes_sent == 0) { + bts.Log.Gen.error($"AsyncTCPConnection_APM._on_send this: {this} sent 0 bytes, throwing exception"); + _handle_error(new ErrorMessage(new ApplicationException("Sent 0 bytes, peer socket is likely shutdown"), + "_on_send")); + return; + } + bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: post endSend this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} " + + $"message: {res.AsyncState} bytes_sent: {bytes_sent} _is_alive: {_is_alive} send_err: {send_err}"); + if (send_err == SocketError.Success) { + // successful send of some bytes + //if (message.offset == message.buffer.Length) { + if (message.fully_transfered()) { + // fully sent, if confirmation needs to be sent out + if (!String.IsNullOrEmpty(message.message_id)) { + message.on_send_del(message.message_id, true); + } + } else { + // there is more to send + _begin_or_resume_sending(message); + } + } else { + _handle_error(new ErrorMessage(send_err, "_on_send")); + } + } catch (Exception ex) { + _handle_error(new ErrorMessage(ex, "_on_send")); + } + } + } + + /// + /// Assembles the heartbeat message, and uses it for + /// future references + /// + /// + private byte[] _build_hb_message() { + if (_cached_hb_message == null) { + BinaryWriter bw = new BinaryWriter(new MemoryStream()); + // the total msg =[msg_sz, msg_type] + int msg_sz = 4; + bw.Write(msg_sz); + bw.Write(HB_MSG_SIGNAL); + _cached_hb_message = (bw.BaseStream as MemoryStream).ToArray(); + } + return _cached_hb_message; + } + + // writes size prefixed msg + private byte[] _build_data_message(byte[] data) { + // TODO, avoid unecessary memory creation (write directly into byte[] using BitConverter) + BinaryWriter bw = new BinaryWriter(new MemoryStream()); + // the total msg =[msg_sz, msg_type, data] + var msg_sz = data.Length + 4; + bw.Write(msg_sz); // +4 for the type of msg + // to indicate data vs hb + bw.Write(DATA_MSG_SIGNAL); // for data + // if we want to id every message, this is the place to do it + bw.Write(data); + return (bw.BaseStream as MemoryStream).ToArray(); + } + + #endregion + + #region receiving data + + private void _read_next_msg() { + _read_size(); + } + + private void _read_size() { + _begin_or_resume_reading(new AsyncReceiveMessage(4, AsyncReceiveMessage.ReadState.reading_size)); + } + + private void _read_data(int data_size) { + // TODO, this could be a problem is the size is too big - maybe read it in chunks (later) + _begin_or_resume_reading(new AsyncReceiveMessage(data_size, AsyncReceiveMessage.ReadState.reading_data)); + } + + private void _begin_or_resume_reading(AsyncReceiveMessage rcv_msg) { + //bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_reading: this: {ToString()} tid: {_tid()} message: {rcv_msg} _is_alive: {_is_alive}"); + if (!_is_alive) + return; + try { + SocketError read_sock_err; + rcv_msg.key = _receive_key++; + var res = _sock.BeginReceive(rcv_msg.buffer, rcv_msg.transfer_ptr, + rcv_msg.remaining_size, + SocketFlags.None, out read_sock_err, + new AsyncCallback(_on_read), + rcv_msg); + if (read_sock_err != SocketError.Success) { + bts.Log.Gen.error($"_begin_or_resume_reading this: {this} error: {read_sock_err} rcv_msg: {rcv_msg}"); + _handle_error(new ErrorMessage(read_sock_err, "_beging_or_resume_reading")); + } else { + // successfully started receiving + if (res != null && res.IsCompleted && res.CompletedSynchronously) { + // make it follow the same code path + Task.Run(() => { + //bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_reading calling _on_send explicitely"); + _on_read(res); + } + ); + } + } + } catch (Exception ex) { + _handle_error(new ErrorMessage(ex, "_beging_or_resume_reading")); + } + } + + private void _on_read(IAsyncResult res) { + AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage; + if (_read_key_map.Contains(rcv_message.key)) { + bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_read key: {rcv_message.key} already handled"); + return; + } + _read_key_map.Add(rcv_message.key); + if (!_is_alive) + return; + try { + SocketError read_err; + int bytes_read = _sock.EndReceive(res, out read_err); + rcv_message.advance_transfer_pointer(bytes_read); + //bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read: this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} bytes_read: {bytes_read} message: {res.AsyncState} _is_alive: {_is_alive} read_err: {read_err}"); + if (bytes_read == 0) { + bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, throwing exception"); + _handle_error(new ErrorMessage(new ApplicationException("Read 0 bytes, peer socket is likely shutdown"), "_on_read")); + return; + } + //return; + if (read_err == SocketError.Success) { + if (rcv_message.fully_transfered()) { + if (rcv_message.is_reading_size()) { + var msg_size = _on_read_size_completely(rcv_message); + if (msg_size != null && msg_size.HasValue) { + _read_data(msg_size.Value); + } + return; + } else { + // data is read + int msg_type = rcv_message.read_int32(); + bts.Log.assert(msg_type == HB_MSG_SIGNAL || msg_type == DATA_MSG_SIGNAL); + if (msg_type == DATA_MSG_SIGNAL) { + _on_read_data_completely(rcv_message); + _read_next_msg(); + return; + return; + } else if (msg_type == HB_MSG_SIGNAL) { + _on_read_hb_completely(rcv_message); + _read_next_msg(); + return; + } + // go back to reading size for a new message + //_read_size(); + } + } else { + // msg has not been fully rd, we have already advanced the transfer ptr + _begin_or_resume_reading(rcv_msg: rcv_message); + } + } else { + _handle_error(new ErrorMessage(read_err, "_on_read")); + } + } catch (Exception ex) { + _handle_error(new ErrorMessage(ex, "_on_read")); + } + } + + #endregion + + private int? _on_read_size_completely(AsyncReceiveMessage msg) { + bts.Log.assert(msg.is_reading_size()); + bts.Log.assert(msg.fully_transfered()); + bts.Log.assert(msg.original_size == 4); + var msg_sz = msg.read_int32(); + //bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely this: {this} msg: {msg} msg_sz: {msg_sz}"); + if (msg_sz > _max_msg_size) { + string err_msg = $"Encountered malformed message, prefixed_size: {msg_sz}"; + _handle_error(new ErrorMessage(new ApplicationException(err_msg), "_on_read")); + return null; + } else { + // sz is determined, now we read that many bytes + //_read_data(msg_sz); + return msg_sz; + } + } + + private void _on_read_hb_completely(AsyncReceiveMessage msg) { + bts.Log.assert(!msg.is_reading_size()); + bts.Log.assert(msg.fully_transfered()); + bts.Log.assert(msg.original_size == 4); + bts.Log.assert(msg.read_int32(0) == HB_MSG_SIGNAL); + if (!_hb_received_yet) { + _hb_received_yet = true; + //bts.Log.Gen.info($"AsyncTCPConnection_APM HB received for the first time from peer {_peer_conn_info}"); + } + _last_hb_received_time = DateTime.Now; + _message_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_conn_info)); + } + private void _on_read_data_completely(AsyncReceiveMessage msg) { + bts.Log.assert(!msg.is_reading_size()); + bts.Log.assert(msg.fully_transfered()); + var type = msg.read_int32(0); + bts.Log.assert(type == DATA_MSG_SIGNAL); + //bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_data_completely this: {this} msg: {msg} type: {type}"); + byte[] data = msg.read_bytes(4); + ///callbacks can be made through thread pool to avoid this + ///thread from being blocked, by this callback + if (!_data_received_yet) { + _data_received_yet = true; + //bts.Log.Gen.info($"AsyncTCPConnection_APM data received for the first time from peer {_peer_conn_info}"); + } + // could be an index here + _message_handler.onData(new RemoteMessage(data, RemoteMessageType.Data, _peer_conn_info)); + } + + #region IDisposable Members + + public void Dispose() { + // socket is closed and disposed here + lock (this) { + if (_is_alive) { + _master_worker.removeWorker(this); + _is_alive = false; + bts.Log.Gen.info($"AsycnTCPConnection.Dispose this: {this} being disposed, last HB was received at {_last_hb_received_time}"); + try { + _sock.Shutdown(SocketShutdown.Both); + } catch (Exception ex) { + String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket"); + if (ex is OutOfMemoryException) + throw ex; + } + _message_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _peer_conn_info)); + } + } + } + + #endregion + + private void _handle_error(ErrorMessage err_msg) { + lock (this) { + Log.Gen.info($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}"); + if (err_msg.oom) + throw err_msg.exception; + Dispose(); + } + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs b/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs index 52ef07ee9..9f91a6f1e 100644 --- a/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs +++ b/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs @@ -8,11 +8,9 @@ using System; using System.Collections.Generic; using System.Diagnostics; -namespace BCG -{ - public class MasterWorker : IWorker - { - +namespace BCG { + public class MasterWorker : IWorker { + private DateTime myLastDoWorkTime = DateTime.MinValue, myLastLogTime = DateTime.MinValue; protected TimeSpan myPeriod = new TimeSpan(0, 0, 0, 0, 100), @@ -23,60 +21,48 @@ namespace BCG readonly Stopwatch timer = new(); private static MasterWorker myThreadDriven = null, myDoWorkDriven = null; - internal MasterWorker() - { + internal MasterWorker() { myInit(); } - protected virtual void myInit() - { } + protected virtual void myInit() { } - public static MasterWorker getInstance (bool threadDriven) - { - lock (typeof(MasterWorker)) - { - if (threadDriven) - { + public static MasterWorker getInstance(bool threadDriven) { + lock (typeof(MasterWorker)) { + if (threadDriven) { // TODO myThreadDriven = myThreadDriven ?? new ThreadDrivenMasterWorker(); return myThreadDriven; return null; - } - else - { + } else { myDoWorkDriven = myDoWorkDriven ?? new MasterWorker(); return myDoWorkDriven; - } + } } } - public virtual void addWorker(IWorker worker, TimeSpan period) - { + public virtual void addWorker(IWorker worker, TimeSpan period) { // BlueEventLog.logEvent(String.Format("MasterWorker : adding worker {0} with period {1} , worker count before adding this = {2}", // worker.ToString(), period.ToString(), myWorkers.Count), // PS3LogLevels.connection); - - myJustAdded.Add(worker, new Pair(period, DateTime.UtcNow)); + + myJustAdded.Add(worker, new Pair(period, DateTime.MinValue)); } - public virtual void removeWorker(IWorker worker) - { + public virtual void removeWorker(IWorker worker) { // BlueEventLog.logEvent(String.Format("MasterWorker : removing worker {0} , worker count before removing this = {1}", // worker.ToString(), myWorkers.Count), // PS3LogLevels.disconnection); - + myJustRemoved.Add(worker, null); } - public virtual void doWork() - { - - foreach (KeyValuePair> p in myJustAdded) - { + public virtual void doWork() { + + foreach (KeyValuePair> p in myJustAdded) { myWorkers.Add(p.Key, p.Value); } - foreach (KeyValuePair> p in myJustRemoved) - { + foreach (KeyValuePair> p in myJustRemoved) { myWorkers.Remove(p.Key); } myJustAdded.Clear(); @@ -85,13 +71,11 @@ namespace BCG { timer.Restart(); //myLogEntries(); - myLastDoWorkTime = DateTime.UtcNow; - foreach (KeyValuePair> p in myWorkers) - { - if (DateTime.UtcNow - p.Value.second >= p.Value.first) - { + myLastDoWorkTime = DateTime.UtcNow; + foreach (KeyValuePair> p in myWorkers) { + if (DateTime.UtcNow - p.Value.second >= p.Value.first) { p.Key.doWork(); - p.Value.second = DateTime.UtcNow; + p.Value.second = DateTime.UtcNow; } } timer.Stop(); @@ -99,14 +83,12 @@ namespace BCG Log.DataFlow.diag($"{GetType()}: took {timer.ElapsedMilliseconds}ms"); } } - + } - protected virtual void myLogEntries () - { - if (DateTime.UtcNow - myLastLogTime > myLogPeriod) - { - myLastLogTime = DateTime.UtcNow; + protected virtual void myLogEntries() { + if (DateTime.UtcNow - myLastLogTime > myLogPeriod) { + myLastLogTime = DateTime.UtcNow; } } @@ -130,7 +112,7 @@ namespace BCG // ///and will be moved over to the main one, in the doWork // lock (this) // { - // myJustAdded.Add(worker, new Pair(period, DateTime.UtcNow)); + // myJustAdded.Add(worker, new Pair(period, DateTime.UtcNow)); // } // } diff --git a/src/DataStreamSuite/PubSub/Misc/StreamUtilityMethods.cs b/src/DataStreamSuite/PubSub/Misc/StreamUtilityMethods.cs index 43a62b14a..89533b27e 100644 --- a/src/DataStreamSuite/PubSub/Misc/StreamUtilityMethods.cs +++ b/src/DataStreamSuite/PubSub/Misc/StreamUtilityMethods.cs @@ -1,6 +1,9 @@ using System; using System.IO; using System.Collections; +using bts; +using System.Threading; +using System.Reflection; namespace BCG { /// @@ -13,8 +16,15 @@ namespace BCG { str = str ?? String.Empty; int len = str.Length; bw.Write(len); - foreach (char c in str) { + foreach (char c_orig in str) { + char c = c_orig > 127 ? 'A' : c_orig; + long p0 = bw.BaseStream.Position; bw.Write(c); + long p1 = bw.BaseStream.Position; + int mv = Convert.ToInt32(p1 - p0); + if (mv > 1) { + Log.assert(false); + } } } diff --git a/src/DataStreamSuite/PubSub/PublisherCore.cs b/src/DataStreamSuite/PubSub/PublisherCore.cs index bd03b5e62..a10daeedf 100644 --- a/src/DataStreamSuite/PubSub/PublisherCore.cs +++ b/src/DataStreamSuite/PubSub/PublisherCore.cs @@ -128,7 +128,7 @@ namespace BCG { case PubSubMessageType.TRANSACTION: bts.Log.assert_d(false, "transaction not supported for now"); break; - case PubSubMessageType.SUBSCRIIBE: + case PubSubMessageType.SUBSCRIBE: case PubSubMessageType.UNSUBSCRIIBE: bts.Log.assert_d(false, $"Incorrect msg: {msg}"); break; diff --git a/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs b/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs index 5835ab4f7..76f3f65a2 100644 --- a/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs +++ b/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs @@ -179,7 +179,7 @@ namespace BCG { string sender = new BlutoSubAddress(message.remoteConnectionInfo.ip, message.remoteConnectionInfo.port).ToString(); switch (message.messageType) { case RemoteMessageType.Data: - // is callback is registered fire it right away, else enqueue it + // if callback is registered fire it right away, else enqueue it if (_data_cb != null) _data_cb(sender, message.message); else @@ -219,9 +219,12 @@ namespace BCG { _connection_map.TryGetValue(p.first, out conn); if (conn != null) { conn.send(p.second); + Log.Gen.info("$ManagedTCPMessageHub._main_message_pump sent n_bytes : {p.second.Length} to: {p.first}"); } else { - string warningMsg = String.Format("sending data to {0} failed, this client might have " + - "disconnected, in which case it will be garbage collected within 60 secs", p.first); + string error_msg = $"sending data of sz: {p.second.Count()} to {p.first} failed, this client might have " + + "disconnected, in which case it will be garbage collected within 60 secs"; + Log.Gen.error(error_msg); + } } } @@ -265,7 +268,7 @@ namespace BCG { int port; conn.getPeerAddress(out ip, out port); string client_url = new BlutoSubAddress(ip, port).ToString(); - bts.Log.Gen.info("ManagedTCPMessageHub._handle_new_connections client_url: {client_url}"); + bts.Log.Gen.info($"ManagedTCPMessageHub._handle_new_connections client_url: {client_url}"); ///to stop it from being garbabge collected immediatly _received_hbs.enqueue(client_url); conn.register(this); diff --git a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs index 4e0474a86..40766243f 100644 --- a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs +++ b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs @@ -1,196 +1,203 @@ -using System; -using System.Threading; - -using bts; - -using Bts.PubSub; - -using Spark.Core; - -namespace BCG { - // This is the main PubSub Server class where it all starts (called by Program.cs). Its constructed and start is called and thats it. - // The only outward facing members of this class is start - // It has an IPubsubMessageHub, that it gets messages from, and updates the PubSubState (which really is all the pubsub server state) - public class PubSubMessageReactor { - private IPubsubMessageHub _msg_hub; - // private delegate void MyMsgHandler(string sender, Stream msg, MetaData m); - // private Dictionary myOperationToHandler = new Dictionary(); - private Thread _msg_polling_thread = null; - // all the data,list of subscribers etc are in the state - private PubSubState _state = new PubSubState(); - private bool myStop = false; - private MasterWorker _mainloop_master_worker = MasterWorker.getInstance(false); - // private double myProcessingWarningThreshold = - // Convert.ToDouble(PubSub3Config.instance.PS3_MessageProcessingWarningThreshold); - // private HighPerformanceTimer myTimer = new HighPerformanceTimer(); - - private delegate void SingleTagTimeSliceDelegate(string tag, double processingTimeSeconds); - private SingleTagTimeSliceDelegate onSingleTagPub; - private static SyncQueue _internal_pub_queue = new SyncQueue(); - //this thing keeps track of whether a given tag is compressed or not. It is used for logging messages. - - /* This delegate allows pubsub components to be notified when the identity ofe the pubsub is known*/ - // private delegate void IdentityFoundDelegate(string identity, bool isRoot); - // private event IdentityFoundDelegate onIdentityFound; - // private void fireIdentityFound(string identity, bool isRoot) { - // if (onIdentityFound != null) - // onIdentityFound(identity, isRoot); - // } - - //private Dictionary myTagCompressionState = new Dictionary(); - - public PubSubMessageReactor(int port_no) { - bts.Log.Gen.info("PubSubMessageReactor.cons System Initialized"); - - _msg_polling_thread = new Thread(new ThreadStart(_polling_thread_func)); - _msg_polling_thread.Priority = ThreadPriority.AboveNormal; - // TODO, fix it - //var port_no = 12345; - _msg_hub = new ManagedTCPMessageHub( - port_no, - TimeSpan.FromMilliseconds(PubSubConfig.Instance.HeartbeatInterval.assert_value()), - PubSubConfig.Instance.MissingHeartbeatsBeforeGc.assert_value(), - TimeSpan.FromMilliseconds(PubSubConfig.Instance.DisposedConnectionCheckPeriod.assert_value())); - - _msg_hub.register_connection_callback(new PubSubCallbacks.PubSubServerConnectionStatusChange(_handle_client_connection)); - - } - - private void _handle_client_connection(string client, PubSubCallbacks.ConnectionStatus conn_status) { - bts.Log.Gen.info($"PubsubMessageReactor._handle_client_connection client: {client} conn_status: {conn_status}"); - ///interested in disconnections only, as connections of no - ///significance until they subscribe, in which case an entry - ///will be made - if (conn_status == PubSubCallbacks.ConnectionStatus.DISCONNECTED) - _state.remove_connection(client); - } - - private void stop() { - myStop = true; - } - - public void start() { - _msg_polling_thread.Start(); - } - - /// - /// This is the main PubSubServer thread. It loops around and read messages from - /// the IPubsubMessageHub. It then deserializes the messages and calls the approrpriate - /// hanlder - /// - private void _polling_thread_func() { - TimeSpan poll_timeout = new TimeSpan(0, 0, 0, 0, 2500); - _msg_hub.activate(); - Action handle_internal_pubs = () => { - var internal_pubs = _internal_pub_queue.dequeueMultipleElements(1, 64000); - foreach (var msg in internal_pubs) { - try { - _handle_pub("Internal", msg, true); - } catch (Exception ex) { - bts.Log.Gen.error($"PubSubMessage>reactor._polling_thread_func ex: {ex}"); - } - } - }; - - while (true) { - try { - var received_data = _msg_hub.get_updates(poll_timeout, 1); - var n_msgs = received_data.Count; - if (n_msgs > 0) { - bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func n_msgs: {n_msgs}"); - } - foreach (var kv in received_data) { - _handle_client_msg(kv.first, kv.second); - } - handle_internal_pubs(); - _mainloop_master_worker.doWork(); - } catch (Exception ex) { - bts.Log.Gen.error($"PubSubMessage>reactor._polling_thread_func ex: {ex}"); - if (ex is OutOfMemoryException) - throw ex; - } - } - } - // all messages sent to the client are handled here - private void _handle_client_msg(string sender, byte[] msg) { - // TODO: reduce object creation, C# has an ObjectPool - PubSubClientMessage ps_client_msg = new PubSubClientMessage(); - ps_client_msg.consruct_deserialize(msg); - bts.Log.Gen.info($"PubSubMessageReator._handle_client_msg ps_client_msg: {ps_client_msg}"); - switch (ps_client_msg.msg_type) { - case PubSubMessageType.SUBSCRIIBE: - _handle_sub(sender, ps_client_msg.sub_msg); - break; - case PubSubMessageType.UNSUBSCRIIBE: - _handle_unsub(sender, ps_client_msg.unsub_msg); - break; - case PubSubMessageType.TRANSACTION: - _handle_transaction(sender, ps_client_msg.transaction_msg); - break; - case PubSubMessageType.PUBLISH: - _handle_pub(sender, ps_client_msg.pub_msg, false); - break; - } - } - - private void _handle_sub(string sender, SubscribeMsg sub_msg) { - bts.Log.Gen.info($"PubSubMessageReactor._handle_sub sender: {sender} sub_msg: {sub_msg}"); - - ///update the state - _state.handle_sub(sub_msg); // state keeps the list of all subscribers - ///to all these new subscriber s, send the existing data - foreach (string tag in sub_msg.tags) { - - byte[] data = _state.get_content(tag); - //ie data has been published once - if (data != null) { - bts.Log.Gen.info($"ManagedTCPMessgeHub._handle_sub tag: {tag} data len: {data.Length}"); - string tagType = _state.getTagType(tag); - var msg_to_send = new PubSubServerMessage() { - tag = tag, - tag_type = tagType, - data = data - }; - _msg_hub.send(sub_msg.connection, ps_msg: msg_to_send); - return; - } - } - - } - private void _handle_pub(string sender, PublishMsg pub_msg, bool is_internal_pub) { - bts.Log.assert(!pub_msg.is_compressed); - bts.Log.Gen.info($"PubSubMessageReactor._handle_pub sender: {sender} pub_msg: {pub_msg} is_internal_pub: {is_internal_pub}"); - // this is where pubsub's data state is updated - _state.update_pub_data(pub_msg); - var sub_connections_for_tag = _state.get_subscriber_connections_for_tag(pub_msg.tag); - bts.Log.Gen.info($"PubSubMessageReactor._handle_pub {sub_connections_for_tag.Count} subs for tag: {pub_msg.tag}"); - bts.PubSubServerMessage msg_to_send = new PubSubServerMessage() { - tag = pub_msg.tag, - tag_type = pub_msg.tag_type, - data = pub_msg.data_to_publish - }; - - ///for all the subscribers, sends the messages over straightaway - foreach (string client_url in sub_connections_for_tag) { - bts.Log.Gen.info($"PubSubMessageReactor._handle_pub sending tag: {pub_msg.tag} to {client_url}"); - _msg_hub.send(client_url, msg_to_send); - } - // if we had to persist the state, this is where we'll do it - } - - private void _handle_transaction(string sender, TransactionMsg transaction_msg) { - bts.Log.assert_d(false, $"transactions are not supported yet msg: {transaction_msg}"); - } - - private void _handle_unsub(string sender, UnsubscribeMsg ubnsub_msg) { - // TODO: check ubnsub_msg.connection - _state.handle_unsub(ubnsub_msg); - } - - public static void internal_publish(PublishMsg msg) { - bts.Log.Gen.diag($"PubsubMessageReactor.internal_publish msg: {msg}"); - bts.Log.Gen.debug_assert(!String.IsNullOrEmpty(msg.tag)); - _internal_pub_queue.enqueue(msg); - } - } -} +using System; +using System.Threading; + +using bts; + +using Bts.PubSub; + +using Spark.Core; + +namespace BCG { + // This is the main PubSub Server class where it all starts (called by Program.cs). Its constructed and start is called and thats it. + // The only outward facing members of this class is start + // It has an IPubsubMessageHub, that it gets messages from, and updates the PubSubState (which really is all the pubsub server state) + public class PubSubMessageReactor { + private IPubsubMessageHub _msg_hub; + // private delegate void MyMsgHandler(string sender, Stream msg, MetaData m); + // private Dictionary myOperationToHandler = new Dictionary(); + private Thread _msg_polling_thread = null; + // all the data,list of subscribers etc are in the state + private PubSubState _state = new PubSubState(); + private bool myStop = false; + private MasterWorker _mainloop_master_worker = MasterWorker.getInstance(false); + // private double myProcessingWarningThreshold = + // Convert.ToDouble(PubSub3Config.instance.PS3_MessageProcessingWarningThreshold); + // private HighPerformanceTimer myTimer = new HighPerformanceTimer(); + + private delegate void SingleTagTimeSliceDelegate(string tag, double processingTimeSeconds); + private SingleTagTimeSliceDelegate onSingleTagPub; + private static SyncQueue _internal_pub_queue = new SyncQueue(); + //this thing keeps track of whether a given tag is compressed or not. It is used for logging messages. + + /* This delegate allows pubsub components to be notified when the identity ofe the pubsub is known*/ + // private delegate void IdentityFoundDelegate(string identity, bool isRoot); + // private event IdentityFoundDelegate onIdentityFound; + // private void fireIdentityFound(string identity, bool isRoot) { + // if (onIdentityFound != null) + // onIdentityFound(identity, isRoot); + // } + + //private Dictionary myTagCompressionState = new Dictionary(); + + public PubSubMessageReactor(int port_no) { + bts.Log.Gen.info("PubSubMessageReactor.cons System Initialized"); + + _msg_polling_thread = new Thread(new ThreadStart(_polling_thread_func)); + _msg_polling_thread.Priority = ThreadPriority.AboveNormal; + // TODO, fix it + //var port_no = 12345; + _msg_hub = new ManagedTCPMessageHub( + port_no, + TimeSpan.FromMilliseconds(PubSubConfig.Instance.HeartbeatInterval.assert_value()), + PubSubConfig.Instance.MissingHeartbeatsBeforeGc.assert_value(), + TimeSpan.FromMilliseconds(PubSubConfig.Instance.DisposedConnectionCheckPeriod.assert_value())); + + _msg_hub.register_connection_callback(new PubSubCallbacks.PubSubServerConnectionStatusChange(_handle_client_connection)); + + } + + private void _handle_client_connection(string client, PubSubCallbacks.ConnectionStatus conn_status) { + bts.Log.Gen.info($"PubsubMessageReactor._handle_client_connection client: {client} conn_status: {conn_status}"); + ///interested in disconnections only, as connections of no + ///significance until they subscribe, in which case an entry + ///will be made + if (conn_status == PubSubCallbacks.ConnectionStatus.DISCONNECTED) + _state.remove_connection(client); + } + + private void stop() { + myStop = true; + } + + public void start() { + _msg_polling_thread.Start(); + } + + /// + /// This is the main PubSubServer thread. It loops around and read messages from + /// the IPubsubMessageHub. It then deserializes the messages and calls the approrpriate + /// hanlder + /// + private void _polling_thread_func() { + TimeSpan poll_timeout = new TimeSpan(0, 0, 0, 0, 2500); + _msg_hub.activate(); + Action handle_internal_pubs = () => { + var internal_pubs = _internal_pub_queue.dequeueMultipleElements(1, 64000); + foreach (var msg in internal_pubs) { + try { + _handle_pub("Internal", msg, true); + } catch (Exception ex) { + bts.Log.Gen.error($"PubSubMessage>reactor._polling_thread_func ex: {ex}"); + } + } + }; + + while (true) { + try { + var received_data = _msg_hub.get_updates(poll_timeout, 1); + var n_msgs = received_data.Count; + if (n_msgs > 0) { + bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func n_msgs: {n_msgs}"); + } + foreach (var kv in received_data) { + _handle_client_msg(kv.first, kv.second); + } + handle_internal_pubs(); + _mainloop_master_worker.doWork(); + } catch (Exception ex) { + bts.Log.Gen.error($"PubSubMessageReactor._polling_thread_func ex: {ex}"); + if (ex is OutOfMemoryException) + throw ex; + } + } + } + // all messages (polled from the message_hub) sent to the client are handled here + private void _handle_client_msg(string sender, byte[] msg) { + // TODO: reduce object creation, C# has an ObjectPool + PubSubClientMessage ps_client_msg = new PubSubClientMessage(); + ps_client_msg.consruct_deserialize(msg); + bts.Log.Gen.info($"PubsubMessageReactor._handle_client_msg sender: {sender} ps_client_msg: {ps_client_msg}"); + switch (ps_client_msg.msg_type) { + case PubSubMessageType.SUBSCRIBE: + _handle_sub(sender, ps_client_msg.sub_msg); + break; + case PubSubMessageType.UNSUBSCRIIBE: + _handle_unsub(sender, ps_client_msg.unsub_msg); + break; + case PubSubMessageType.TRANSACTION: + _handle_transaction(sender, ps_client_msg.transaction_msg); + break; + case PubSubMessageType.PUBLISH: + _handle_pub(sender, ps_client_msg.pub_msg, false); + break; + } + } + + private void _handle_sub(string sender, SubscribeMsg sub_msg) { + //TODO: we make this correction here, since non managed clients are ntosending this correctly + bts.Log.Gen.info($"PubSubMessageReactor._handle_sub sender: {sender} sub_msg: {sub_msg}"); + if (String.IsNullOrEmpty(sub_msg.connection) || sub_msg.connection.Contains("unset")) { + sub_msg.connection = sender; + } + bts.Log.Gen.info($"PubSubMessageReactor._handle_sub(corrected) sender: {sender} sub_msg: {sub_msg}"); + + ///update the state + _state.handle_sub(sub_msg); // state keeps the list of all subscribers + ///to all these new subscriber s, send the existing data + foreach (string tag in sub_msg.tags) { + byte[] data = _state.get_content(tag); + //ie data has been published once + if (data != null) { + bts.Log.Gen.info($"ManagedTCPMessgeHub._handle_sub get_content tag: {tag} data len: {data.Length}"); + string tagType = _state.getTagType(tag); + var msg_to_send = new PubSubServerMessage() { + tag = tag, + tag_type = tagType, + data = data + }; + bts.Log.Gen.info($"ManagedTCPMessgeHub._handle_sub sending msg_to_send: {msg_to_send} connection: {sub_msg.connection}"); + _msg_hub.send(sub_msg.connection, ps_msg: msg_to_send); + return; + } else { + bts.Log.Gen.error($"ManagedTCPMessgeHub._handle_sub could not retreive data for sub_msg: {sub_msg}"); + } + } + + } + private void _handle_pub(string sender, PublishMsg pub_msg, bool is_internal_pub) { + bts.Log.assert(!pub_msg.is_compressed); + //bts.Log.Gen.info($"PubSubMessageReactor._handle_pub sender: {sender} pub_msg: {pub_msg} is_internal_pub: {is_internal_pub}"); + // this is where pubsub's data state is updated + _state.update_pub_data(pub_msg); + var sub_connections_for_tag = _state.get_subscriber_connections_for_tag(pub_msg.tag); + //bts.Log.Gen.info($"PubSubMessageReactor._handle_pub {sub_connections_for_tag.Count} subs for tag: {pub_msg.tag}"); + bts.PubSubServerMessage msg_to_send = new PubSubServerMessage() { + tag = pub_msg.tag, + tag_type = pub_msg.tag_type, + data = pub_msg.data_to_publish + }; + + ///for all the subscribers, sends the messages over straightaway + foreach (string client_url in sub_connections_for_tag) { + //bts.Log.Gen.info($"PubSubMessageReactor._handle_pub sending tag: {pub_msg.tag} to {client_url}"); + _msg_hub.send(client_url, msg_to_send); + } + // if we had to persist the state, this is where we'll do it + } + + private void _handle_transaction(string sender, TransactionMsg transaction_msg) { + bts.Log.assert_d(false, $"transactions are not supported yet msg: {transaction_msg}"); + } + + private void _handle_unsub(string sender, UnsubscribeMsg ubnsub_msg) { + // TODO: check ubnsub_msg.connection + _state.handle_unsub(ubnsub_msg); + } + + public static void internal_publish(PublishMsg msg) { + bts.Log.Gen.diag($"PubsubMessageReactor.internal_publish msg: {msg}"); + bts.Log.Gen.debug_assert(!String.IsNullOrEmpty(msg.tag)); + _internal_pub_queue.enqueue(msg); + } + } +} diff --git a/src/DataStreamSuite/PubSub/Server/PubSubState.cs b/src/DataStreamSuite/PubSub/Server/PubSubState.cs index 6f3519748..8d4abd3c5 100644 --- a/src/DataStreamSuite/PubSub/Server/PubSubState.cs +++ b/src/DataStreamSuite/PubSub/Server/PubSubState.cs @@ -233,7 +233,7 @@ namespace BCG { return null; var rv = _tag_to_data_map[tag].get_current_content(); var rv_txt = StreamUtilityMethods.first_n_bytes_to_string(rv, 4); - bts.Log.Gen.info("PubSubState.get_content tag: {tag} content len: {rv.Length} rv: {rv_txt}"); + bts.Log.Gen.info($"PubSubState.get_content tag: {tag} content len: {rv.Length} rv: {rv_txt}"); return rv; } diff --git a/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs b/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs index 1d5d919ef..882f28326 100644 --- a/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs +++ b/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs @@ -114,7 +114,7 @@ namespace BCG { private void _subscribe(params string[] tag_args) { // coonnection and app bts.PubSubClientMessage msg = new bts.PubSubClientMessage() { - msg_type = bts.PubSubMessageType.SUBSCRIIBE, + msg_type = bts.PubSubMessageType.SUBSCRIBE, sub_msg = new bts.SubscribeMsg() { connection = _connection.client_url, subscribing_app = ClientInfo.thisClient.processString, diff --git a/src/DataStreamSuite/PubSub/Testing/SimpleTCPClient.cs b/src/DataStreamSuite/PubSub/Testing/SimpleTCPClient.cs index 449a5883e..c5fff1d8b 100644 --- a/src/DataStreamSuite/PubSub/Testing/SimpleTCPClient.cs +++ b/src/DataStreamSuite/PubSub/Testing/SimpleTCPClient.cs @@ -20,9 +20,12 @@ namespace BCG { private readonly UInt32 _msg_sz = 1024;// * 1024; private readonly int _max_n_msgs = Int32.MaxValue; private bool _send_flag = true; - int _send_wait_period_ms = 1000; + int _send_wait_period_ms = 5000; + Action _listener; - public SimpleTCPClient(string server_ip, int server_port) { + public SimpleTCPClient(string server_ip, int server_port, UInt32 msg_sz, int send_wait_period_ms) { + _msg_sz = msg_sz; + _send_wait_period_ms = send_wait_period_ms; _server_addr = new BlutoSubAddress(server_ip, server_port); } @@ -34,11 +37,17 @@ namespace BCG { bts.AsyncTCPConnector_APM.connect_to(_server_addr, this); } + public void register_listener(Action cb) { + _listener += cb; + } public void onConnection(ITCPConnection connection) { string ip; int port; connection.getPeerAddress(out ip, out port); bts.Log.Gen.info($"SimpleTCPClientAsync.onConnection connected to server ip: {ip} port: {port}, starting task to send data in a loop"); + if (_listener != null) { + _listener.Invoke($"Connected to server ip: {ip} port {port}"); + } connection.register(this); // start sending data _send_flag = true; @@ -57,12 +66,14 @@ namespace BCG { bts.Log.Gen.info("Sending msg: {i} sz: {buff.Length}"); System.Threading.Thread.Sleep(_send_wait_period_ms); connection.send(buff); + _listener.Invoke($"Data sent, sz: {buff.Length}"); } } public void onData(RemoteMessage msg) { _m_msgs_received++; Console.WriteLine($"SimpleTCPClientAsync.onData n_msgs: {_m_msgs_received} msg: {msg}"); + _listener.Invoke($"Message received msg: {msg}"); if (msg.messageType == RemoteMessageType.Disconnection) { _send_flag = false; _connect(); diff --git a/src/DataStreamSuite/PubSub/documentation/readme.txt b/src/DataStreamSuite/PubSub/documentation/readme.txt index 17b956ca9..dc7a27efc 100644 --- a/src/DataStreamSuite/PubSub/documentation/readme.txt +++ b/src/DataStreamSuite/PubSub/documentation/readme.txt @@ -37,7 +37,7 @@ Dictionary and Dictionary SingleTagData has the tag and IPubSubServerTagHandler (or byte[]) (IPubSubServerTagHandler holds the data state) for streams the IPubSubServerTagHandler is the PubsubServerTagHandler, which is designed for multiple tags and has a IDictionary -TagStateBank containes the PubsubDeserializationController, which knows how to deserialize the stream publishes and build state, ad simialirly produce the state on demand +TagStateBank containes the PubsubDeserial izationController, which knows how to deserialize the stream publishes and build state, ad simialirly produce the state on demand PubsubTagHandlerFactory returns a IPubSubServerTagHandler for each tag, called by the server state handler, SingleTagData and publisher tag state, AllPublishedTagState SingleTagConnectionData contains connection vs sub, and connection vs pub @@ -75,4 +75,43 @@ the second arg is IBlueRowCollection[] because in case of op = StreamOpCode.mult (BlueDataSerializer.serialize's comment has the serialization layout) For each serializer we have 2 matching deserializers, IPubsubDeserializer (for PubSubservr) and IClientDeserializer (for clients subscribing to streams) -For UniversalSerializer, we have PubsubUniversalDeserializer and ClientUniversalDeserializer \ No newline at end of file +For UniversalSerializer, we have PubsubUniversalDeserializer and ClientUniversalDeserializer + +------------------------------------------------------------------- +TCP publishing: +To run the base TCP layer run PubSubServer as an echo server, +(set run_echo_tcp_server_only in spark\src\DataStreamSuite\PubSubServer\Program.cs) +to run a client that sends a tcp message tot he echo server, run +SimpleTCPEchoStream + + +Pubsub publishing: +----------------- +PubSubTestingStream (bounces pubs across the pubsub server) +mutableAppend(pubSubTesting(100,10000000,100000,"test_1"), +pubSubTesting(250,10000000,100000,"test_2")) + + +Stream publishing: +------------------ +PublisherStream (publishesa any stream to the pubsubserver) +r=second(100) +r=project(r,"id") +r=math(r,"x = id*id") +publisher(r, "test_x") + + +tracking message into the pubsubserver +-------------------------------------- +connection: +ServerToClientManagedTCPConnectionAcceptor._on_accept + var log_msg = $"ServerToClientManagedTCPConnectionAcceptor._on_accept called port: {_port} state: {res.AsyncState}"; + Log.Gen.info(log_msg); + +data: +ManagedTCPMessageHub.onData +bts.Log.Gen.info($"ManagedTCPMessgageHub.onData recv msg: {message}"); +PubSubMessageReactor._polling_thread_func +bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func n_msgs: {n_msgs}"); +PubSubMessageReactor._handle_client_msg +bts.Log.Gen.info($"PubSubMessageReator._handle_client_msg ps_client_msg: {ps_client_msg}"); diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubsubTagStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubsubTagStream.cs index c1d131efe..a91853a09 100644 --- a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubsubTagStream.cs +++ b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubsubTagStream.cs @@ -1,76 +1,76 @@ -using System; -using System.IO; -using System.Text; -using System.Collections; +// using System; +// using System.IO; +// using System.Text; +// using System.Collections; -namespace BCG -{ - /// - /// Gives the folowing information about every tag in the system : - /// - last update - /// - size of last update - /// This can be useful for watching non stream based tag, for which - /// theres no GUI. However, the stream subscribes to that tag, so - /// the subscription cost is there - /// - [DataStreamUserRespresentation ("tag")] - public class PubsubTagStream : ViewStream - { - private string[] myTags; - private ManagedSubscriber mySub; - private BlueTable myRows = new BlueTable (new BlueTableSchema ("Pubsubtags", - new ColumnInfo ("id", 0, typeof(int)), - new ColumnInfo ("tag", 1, typeof(string)), - new ColumnInfo ("size", 2, typeof(int)), - new ColumnInfo ("lastUpdate", 3, typeof(DateTime)))); - private IDictionary myTagRows = new Hashtable (); - private int myID = 0; +// namespace BCG +// { +// /// +// /// Gives the folowing information about every tag in the system : +// /// - last update +// /// - size of last update +// /// This can be useful for watching non stream based tag, for which +// /// theres no GUI. However, the stream subscribes to that tag, so +// /// the subscription cost is there +// /// +// [DataStreamUserRespresentation ("tag")] +// public class PubsubTagStream : ViewStream +// { +// private string[] myTags; +// private ManagedSubscriber mySub; +// private BlueTable myRows = new BlueTable (new BlueTableSchema ("Pubsubtags", +// new ColumnInfo ("id", 0, typeof(int)), +// new ColumnInfo ("tag", 1, typeof(string)), +// new ColumnInfo ("size", 2, typeof(int)), +// new ColumnInfo ("lastUpdate", 3, typeof(DateTime)))); +// private IDictionary myTagRows = new Hashtable (); +// private int myID = 0; - public PubsubTagStream(string tags) - : this(ViewStreamDatabase.instance, tags) - {} +// public PubsubTagStream(string tags) +// : this(ViewStreamDatabase.instance, tags) +// {} - public PubsubTagStream(ViewStreamDatabase vsd, string tags) - { - myTags = tags.Split(','); - mySub = ManagedSubscriber.instance; - vsd.addWorker(mySub); - } +// public PubsubTagStream(ViewStreamDatabase vsd, string tags) +// { +// myTags = tags.Split(','); +// mySub = ManagedSubscriber.instance; +// vsd.addWorker(mySub); +// } - protected override void myOnFirstSubscriber() - { - foreach (string t in myTags) - mySub.subscribe(t.Trim(), new RawDataHandler(myRawDataHandler)); - } +// protected override void myOnFirstSubscriber() +// { +// foreach (string t in myTags) +// mySub.subscribe(t.Trim(), new RawDataHandler(myRawDataHandler)); +// } - protected override void myOnLastSubscriber() - { - foreach (string t in myTags) - mySub.unsubscribe(t.Trim(), new RawDataHandler(myRawDataHandler)); - } +// protected override void myOnLastSubscriber() +// { +// foreach (string t in myTags) +// mySub.unsubscribe(t.Trim(), new RawDataHandler(myRawDataHandler)); +// } - private void myRawDataHandler(string tag, Stream s) - { - BlueTable dels = new BlueTable (myRows.schema), - adds = new BlueTable (myRows.schema); - if (!myTagRows.Contains (tag)) - { - MutableBlueRow r = new MutableBlueRow (myRows.schema, myID++, tag, s.Length, DateTime.Now); - myRows.addRow (r); - myTagRows.Add (tag, r); - adds.addRow (r); - myFireEvent (new DataPacket (myRows, adds, dels)); - } - else - { - MutableBlueRow r = myTagRows[tag] as MutableBlueRow; - IBlueRow delRow = new BlueRow (r); - r.setValue ("size", s.Length); - r.setValue ("lastUpdate", DateTime.Now); - dels.addRow (delRow); - adds.addRow (r); - myFireEvent (new DataPacket (myRows, adds, dels)); - } - } - } -} +// private void myRawDataHandler(string tag, Stream s) +// { +// BlueTable dels = new BlueTable (myRows.schema), +// adds = new BlueTable (myRows.schema); +// if (!myTagRows.Contains (tag)) +// { +// MutableBlueRow r = new MutableBlueRow (myRows.schema, myID++, tag, s.Length, DateTime.Now); +// myRows.addRow (r); +// myTagRows.Add (tag, r); +// adds.addRow (r); +// myFireEvent (new DataPacket (myRows, adds, dels)); +// } +// else +// { +// MutableBlueRow r = myTagRows[tag] as MutableBlueRow; +// IBlueRow delRow = new BlueRow (r); +// r.setValue ("size", s.Length); +// r.setValue ("lastUpdate", DateTime.Now); +// dels.addRow (delRow); +// adds.addRow (r); +// myFireEvent (new DataPacket (myRows, adds, dels)); +// } +// } +// } +// }