Compare commits

...
Author SHA1 Message Date
Taha d9e4f9ff9e Fixing mangled merge 2023-09-30 12:18:23 -04:00
Taha e30a762969 Adding tests 2023-09-30 12:08:51 -04:00
Taha 83d016d46a Streams can be published across PubSub now 2023-09-30 12:07:10 -04:00
Taha 0039d08b5b PubSub is working now 2023-09-30 12:03:30 -04:00
Taha 74627f9a8a AsyncTCPCOnnection2 done 2023-09-30 12:01:58 -04:00
Taha 0b18e95a77 Both SImpleTCPClient and SimpleTCPServer use non async calls now 2023-09-30 12:01:58 -04:00
Taha 57c56f05c2 Merging, and adding test project 2023-09-30 12:01:05 -04:00
66 changed files with 3466 additions and 2119 deletions

No files matched your search

Submodule
+1
Submodule docker/linux added at 6537812262.
+1
Submodule docker/windows added at de7d40c09d.
@@ -1,72 +1,68 @@
using System; using System;
using System.Collections; using System.Collections;
using System.Collections.Generic;
using BCG.Utilities; using BCG.Utilities;
namespace BCG namespace BCG {
{
/// <summary> /// <summary>
/// This class will be used by the clients to publish data to the pubsub /// This class will be used by the clients to publish tabular data to the pubsub
/// </summary> /// </summary>
public class PubTableBroker public class PubTableBroker {
{ protected PublisherCore _publisher_core;
protected PublisherCore myPublisherCore; protected IDictionary<string, IPubTablePublisher> _publisher_map = new Dictionary<string, IPubTablePublisher>();
protected IDictionary myPublisherMap = new Hashtable (); protected Dictionary<string, IBlueTableSchema> _schema_map = new Dictionary<string, IBlueTableSchema>();
protected IDictionary mySchemaMap = new Hashtable ();
protected bool myWaitingForFirstOp = false; protected bool myWaitingForFirstOp = false;
public PubTableBroker () public PubTableBroker()
: this (new PublisherCore()) : this(new PublisherCore()) { }
{}
public PubTableBroker (PublisherCore publisher) public PubTableBroker(PublisherCore publisher) { _publisher_core = publisher; }
{myPublisherCore = publisher;}
public PubTableBroker (IBlueRowCollection[] tablesToPublish) public PubTableBroker(IBlueRowCollection[] tablesToPublish)
: this (new PublisherCore(), tablesToPublish) : this(new PublisherCore(), tablesToPublish) { }
{}
public PubTableBroker (PublisherCore publisher, IBlueRowCollection[] tablesToPublish) public PubTableBroker(PublisherCore publisher, IBlueRowCollection[] tablesToPublish) {
{ _publisher_core = publisher;
myPublisherCore = publisher;
myPublisherCore.beginTransaction (); _publisher_core.beginTransaction();
foreach (IBlueRowCollection table in tablesToPublish) foreach (IBlueRowCollection table in tablesToPublish)
createTable (table); createTable(table);
myPublisherCore.commitTransaction (); _publisher_core.commitTransaction();
} }
public void beginTransaction () public void beginTransaction() {
{
myWaitingForFirstOp = true; myWaitingForFirstOp = true;
} }
public void commitTransaction () public void commitTransaction() {
{ bts.Log.Gen.info("PubTableBroker.commitTransaction");
if (myWaitingForFirstOp) if (myWaitingForFirstOp)
myWaitingForFirstOp = false; myWaitingForFirstOp = false;
else else
myPublisherCore.commitTransaction (); _publisher_core.commitTransaction();
} }
public void rollbackTransaction () public void rollbackTransaction() {
{ bts.Log.Gen.info("PubTableBroker.rollbackTransaction");
if (myWaitingForFirstOp) if (myWaitingForFirstOp)
myWaitingForFirstOp = false; myWaitingForFirstOp = false;
else else
myPublisherCore.abortTransaction (); _publisher_core.abortTransaction();
} }
public virtual void createTable (IBlueRowCollection newTable) public virtual void createTable(IBlueRowCollection newTable) {
{ bts.Log.Gen.info($"PubTableBroker.createTable table: {newTable.schema.tableName}");
myCheckTransaction(); myCheckTransaction();
if (myPublisherMap [newTable.schema.tableName.ToLower()] == null) IPubTablePublisher table_publisher = null;
{ var name_key = newTable.schema.tableName.ToLower();
myPublisherMap [newTable.schema.tableName.ToLower()] = myGetIPubTablePublisher (); if (!_publisher_map.TryGetValue(name_key, out table_publisher)) {
mySchemaMap [newTable.schema.tableName.ToLower()] = newTable.schema; table_publisher = _create_pubtable_publisher(name_key);
_publisher_map.Add(name_key, table_publisher);
_schema_map.Add(name_key, newTable.schema);
} }
(myPublisherMap[newTable.schema.tableName.ToLower()] as IPubTablePublisher).createTable (newTable); table_publisher.createTable(newTable);
} }
/// <summary> /// <summary>
@@ -74,13 +70,14 @@ namespace BCG
/// will be thrown /// will be thrown
/// </summary> /// </summary>
/// <param name="r"></param> /// <param name="r"></param>
public virtual void addRow (IBlueRow r) public virtual void addRow(IBlueRow r) {
{
myCheckTransaction(); myCheckTransaction();
if (myPublisherMap[r.schema.tableName.ToLower()] == null) IPubTablePublisher table_publisher = null;
throw new ApplicationException ("Row can not be added, as the table that has not been created yet"); var name_key = r.schema.tableName.ToLower();
if (!_publisher_map.TryGetValue(name_key, out table_publisher))
throw new ApplicationException("Row can not be added, as the table that has not been created yet");
else else
(myPublisherMap [r.schema.tableName.ToLower()] as IPubTablePublisher).addRow (r); table_publisher.addRow(r);
} }
/// <summary> /// <summary>
@@ -88,13 +85,14 @@ namespace BCG
/// will be thrown /// will be thrown
/// </summary> /// </summary>
/// <param name="r"></param> /// <param name="r"></param>
public virtual void deleteRow (IBlueRow r) public virtual void deleteRow(IBlueRow r) {
{
myCheckTransaction(); myCheckTransaction();
if (myPublisherMap[r.schema.tableName.ToLower()] == null) IPubTablePublisher table_publisher = null;
throw new ApplicationException ("Row can not be removed, as the table has not been created yet"); var name_key = r.schema.tableName.ToLower();
if (!_publisher_map.TryGetValue(name_key, out table_publisher))
throw new ApplicationException("Row can not be removed, as the table has not been created yet");
else else
(myPublisherMap [r.schema.tableName.ToLower()] as IPubTablePublisher).deleteRow (r); table_publisher.deleteRow(r);
} }
/// <summary> /// <summary>
@@ -102,13 +100,14 @@ namespace BCG
/// will be thrown /// will be thrown
/// </summary> /// </summary>
/// <param name="r"></param> /// <param name="r"></param>
public virtual void changeRow (IBlueRow r) public virtual void changeRow(IBlueRow r) {
{
myCheckTransaction(); myCheckTransaction();
if (myPublisherMap[r.schema.tableName.ToLower()] == null) IPubTablePublisher table_publisher = null;
throw new ApplicationException ("Row can not be changed, as the table has not been created yet"); var name_key = r.schema.tableName.ToLower();
if (!_publisher_map.TryGetValue(name_key, out table_publisher))
throw new ApplicationException("Row can not be changed, as the table has not been created yet");
else else
(myPublisherMap [r.schema.tableName.ToLower()] as IPubTablePublisher).changeRow (r); table_publisher.changeRow(r);
} }
/// <summary> /// <summary>
@@ -116,40 +115,37 @@ namespace BCG
/// will be thrown /// will be thrown
/// </summary> /// </summary>
/// <param name="r"></param> /// <param name="r"></param>
public virtual void makeBatchChanges (IBlueRowCollection newRows, public virtual void makeBatchChanges(IBlueRowCollection newRows,
IBlueRowCollection deletedRows, IBlueRowCollection deletedRows,
IBlueRowCollection addedRows) IBlueRowCollection addedRows) {
{
myCheckTransaction(); myCheckTransaction();
if (myPublisherMap[newRows.schema.tableName.ToLower()] == null) IPubTablePublisher table_publisher = null;
throw new ApplicationException ("Batch changes can not be made, as the table has not been created yet"); var name_key = newRows.schema.tableName.ToLower();
if (!_publisher_map.TryGetValue(name_key, out table_publisher))
throw new ApplicationException("Batch changes can not be made, as the table has not been created yet");
else else
(myPublisherMap [newRows.schema.tableName.ToLower()] as IPubTablePublisher).batchRemoveAdd (deletedRows, addedRows); table_publisher.batchRemoveAdd(deletedRows, addedRows);
} }
public virtual void deleteTableRows (string tableName) public virtual void deleteTableRows(string tableName) {
{
myCheckTransaction(); myCheckTransaction();
(myPublisherMap [tableName.ToLower()] as IPubTablePublisher).createTable ( (_publisher_map[tableName.ToLower()] as IPubTablePublisher).createTable(
new BlueTable (mySchemaMap[tableName.ToLower()] as IBlueTableSchema)); new BlueTable(_schema_map[tableName.ToLower()] as IBlueTableSchema));
} }
public int getNumberOfWaitingMessages() { return myPublisherCore.getNumberOfWaitingMessages(); } public int getNumberOfWaitingMessages() { return _publisher_core.getNumberOfWaitingMessages(); }
public void clearTable(string tableName)
{myPublisherCore.clearState(PubStreamPrefixer.prefix(tableName));}
protected void myCheckTransaction() public void clearTable(string tableName) { _publisher_core.clearState(PubStreamPrefixer.prefix(tableName)); }
{
protected void myCheckTransaction() {
if (myWaitingForFirstOp) if (myWaitingForFirstOp)
myPublisherCore.beginTransaction (); _publisher_core.beginTransaction();
myWaitingForFirstOp = false; myWaitingForFirstOp = false;
} }
protected virtual IPubTablePublisher myGetIPubTablePublisher () protected virtual IPubTablePublisher _create_pubtable_publisher(string tag) {
{ return new PubTablePublisher(tag, _publisher_core);
return new PubTablePublisher (myPublisherCore);
} }
} }
} }
@@ -4,62 +4,51 @@ using System.Data;
using BCG.Utilities; using BCG.Utilities;
namespace BCG namespace BCG {
{ public class ViewPublisher : IDisposable {
public class ViewPublisher : IDisposable protected ViewPublisher() { }
{
protected ViewPublisher () {}
/// If an name IsNullOrEmpty then it uses the name /// If an name IsNullOrEmpty then it uses the name
/// in the schema (be careful with that because this /// in the schema (be careful with that because this
/// could lead to double publishes if the schema name /// could lead to double publishes if the schema name
/// is not uniquely chosen) /// is not uniquely chosen)
public ViewPublisher(string name, PubTableBroker br, ViewStream vs) public ViewPublisher(string name, PubTableBroker br, ViewStream vs) {
{
myName = name; myName = name;
myView = new TableStream(name); // TODOds String.IsNullOrEmpty(name) ? vs : new ProjectStream(vs, "!", myName); myView = vs;//String.IsNullOrEmpty(name) ? vs : new ProjectStream(vs, "!", myName);
myBroker = br; myBroker = br;
myFirstTime = true; myFirstTime = true;
myView.onChange += new DataStream.ChangeDelegate(myChangeHandler); myView.onChange += new DataStream.ChangeDelegate(myChangeHandler);
} }
public void Dispose() public void Dispose() {
{
myView.onChange -= new DataStream.ChangeDelegate(myChangeHandler); myView.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
if (!String.IsNullOrEmpty(myName)) if (!String.IsNullOrEmpty(myName))
myBroker.clearTable(myName); myBroker.clearTable(myName);
} }
private void myChangeHandler(object o) private void myChangeHandler(object o) {
{ DataPacket dp = (DataPacket)o;
DataPacket dp = (DataPacket) o; if (myFirstTime) {
if (myFirstTime)
{
myName = String.IsNullOrEmpty(myName) ? dp.schema.tableName : myName; myName = String.IsNullOrEmpty(myName) ? dp.schema.tableName : myName;
myLastRowCount = dp.newRows.Count; myLastRowCount = dp.newRows.Count;
myBroker.createTable(dp.newRows); myBroker.createTable(dp.newRows);
myFirstTime = false; myFirstTime = false;
} } else {
else if (((dp.addedRows.Count + dp.deletedRows.Count) != 0)) {
{ myLastRowCount = dp.newRows.Count;
if (((dp.addedRows.Count + dp.deletedRows.Count) != 0)) myBroker.makeBatchChanges(dp.newRows, dp.deletedRows, dp.addedRows);
{
myLastRowCount = dp.newRows.Count;
myBroker.makeBatchChanges(dp.newRows,dp.deletedRows, dp.addedRows);
} }
} }
} }
public int getLastRowPublishedCount() public int getLastRowPublishedCount() { return myLastRowCount; }
{return myLastRowCount;}
public bool hasPublished() public bool hasPublished() { return !myFirstTime; }
{ return !myFirstTime; }
private int myLastRowCount = -1; private int myLastRowCount = -1;
private string myName; private string myName;
private PubTableBroker myBroker; private PubTableBroker myBroker;
protected ViewStream myView; protected ViewStream myView;
private bool myFirstTime; private bool myFirstTime;
} }
} }
@@ -112,6 +112,34 @@ namespace BCG
#endregion #endregion
public static void print_table(this IBlueRowCollection table, Func<string, int> get_col_width)
{
// Print column headers
Console.WriteLine($"{table.schema.tableName}______________________________________");
foreach (var col_name in table.schema.columnNames)
{
int col_width = get_col_width(col_name);
Console.Write($"{col_name.PadRight(col_width)} ");
}
Console.WriteLine();
Console.WriteLine("______________________________________");
// Print table data
foreach (IBlueRow row in table)
{
foreach (string col_name in table.schema.columnNames)
{
int col_width = get_col_width(col_name);
object row_val = row[col_name];
string row_val_str = row_val.ToString();
if (row_val != null)
{
Console.Write($"{row_val_str.PadRight(col_width)} ");
}
}
Console.WriteLine();
}
}
private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type> private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
{ {
{ TypeCode.Boolean, typeof(bool) }, { TypeCode.Boolean, typeof(bool) },
@@ -895,12 +923,12 @@ namespace BCG
out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows)
{ {
Func<IBlueRow, IBlueRow, bool> rows_equal = (r1, r2) => Func<IBlueRow, IBlueRow, bool> rows_equal = (r1, r2) =>
{ {
foreach (int col_no in col_indices) foreach (int col_no in col_indices)
if (!(r1[col_no] ?? DBNull.Value).Equals(r2[col_no] ?? DBNull.Value)) if (!(r1[col_no] ?? DBNull.Value).Equals(r2[col_no] ?? DBNull.Value))
return false; return false;
return true; return true;
}; };
BlueTable del_rows = new BlueTable(new_rows.schema), add_rows = new BlueTable(new_rows.schema); BlueTable del_rows = new BlueTable(new_rows.schema), add_rows = new BlueTable(new_rows.schema);
// adds + changes // adds + changes
@@ -134,18 +134,17 @@ namespace BCG
/// <param name="row"></param> /// <param name="row"></param>
public virtual void addRow(IBlueRow row) public virtual void addRow(IBlueRow row)
{ {
object val = row["id"]; object id = row["id"];
if (val == null || val == DBNull.Value) if (id == null || id == DBNull.Value)
throw new BlueDataException("The ID of the row being added is null/DNBull"); throw new BlueDataException("The ID of the row being added is null/DNBull");
try try
{ {
myIdIndex.Add(val, row); myIdIndex.Add(id, row);
} }
catch (ArgumentException) catch (ArgumentException)
{ {
throw new BlueDataException("ID already exists in the table " + mySchema.tableName + throw new BlueDataException($"ID {id} already exists in the table: {mySchema.tableName} , duplication of field ID not allowed");
" , duplication of field ID not allowed");
} }
} }
+279 -279
View File
@@ -1,316 +1,316 @@
using System; // using System;
using System.Collections; // using System.Collections;
using System.Data; // using System.Data;
using BCG.Utilities.General; // using BCG.Utilities.General;
namespace BCG // namespace BCG
{ // {
[Serializable] // [Serializable]
public class DataPacket : ICloneable, IDataPacket // public class DataPacket : ICloneable, IDataPacket
{ // {
public class BlueRowIdComparator // public class BlueRowIdComparator
{ // {
private string[] myIdColumns = new string[]{"id"}; // private string[] myIdColumns = new string[]{"id"};
public BlueRowIdComparator(string[] idCols) // public BlueRowIdComparator(string[] idCols)
{ // {
if ( idCols.Length != 0 ) // if ( idCols.Length != 0 )
{ // {
myIdColumns = idCols; // myIdColumns = idCols;
} // }
} // }
public bool blueRowIdEqualEvaluator(object a, object b)
{
if ( a == null || b == null )
{
return false;
}
if ( !(a is IBlueRow) || !(b is IBlueRow) ) // public bool blueRowIdEqualEvaluator(object a, object b)
{ // {
return false; // if ( a == null || b == null )
} // {
// return false;
// }
IBlueRow lhs = (IBlueRow)a; // if ( !(a is IBlueRow) || !(b is IBlueRow) )
IBlueRow rhs = (IBlueRow)b; // {
// return false;
bool result = true; // }
foreach(string idColumn in myIdColumns)
{
result = result && lhs[idColumn].Equals(rhs[idColumn]);
if ( !result )
break;
}
return result; // IBlueRow lhs = (IBlueRow)a;
} // IBlueRow rhs = (IBlueRow)b;
}
public IBlueRowCollection newRows, addedRows, deletedRows; // bool result = true;
public DataPacket () {} // foreach(string idColumn in myIdColumns)
public DataPacket (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows) // {
{ // result = result && lhs[idColumn].Equals(rhs[idColumn]);
this.newRows = newRows; // if ( !result )
this.addedRows = addedRows; // break;
this.deletedRows = deletedRows; // }
}
public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) // return result;
{ // }
new_rows = newRows; // }
added_rows = addedRows;
deleted_rows = deletedRows;
}
public IBlueRowCollection getTrulyAddedRows(params string[] idColumns) // public IBlueRowCollection newRows, addedRows, deletedRows;
{ // public DataPacket () {}
BlueTable truly = new BlueTable(addedRows.schema); // public DataPacket (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows)
BlueRowIdComparator c = new BlueRowIdComparator(idColumns); // {
// this.newRows = newRows;
// this.addedRows = addedRows;
// this.deletedRows = deletedRows;
// }
CollectionHelpers.EqualDelegate ed = // public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows)
new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); // {
// new_rows = newRows;
// added_rows = addedRows;
// deleted_rows = deletedRows;
// }
ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed); // public IBlueRowCollection getTrulyAddedRows(params string[] idColumns)
// {
foreach(IBlueRow r in ta) // BlueTable truly = new BlueTable(addedRows.schema);
{ // BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
truly.addRow(r);
}
return truly;
}
public virtual IBlueRowCollection getTrulyAddedRows () // CollectionHelpers.EqualDelegate ed =
{ // new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
BlueTable retVal = new BlueTable (schema);
foreach (IBlueRow r in addedRows)
if (!deletedRows.containsID ((int)r["id"]))
retVal.addRow (r);
return retVal;
}
public virtual IBlueRowCollection getTrulyDeletedRows () // ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
{
BlueTable retVal = new BlueTable (schema);
foreach (IBlueRow r in deletedRows)
if (!addedRows.containsID ((int)r["id"]))
retVal.addRow (r);
return retVal;
}
public virtual IBlueRowCollection getTrulyChangedRows () // foreach(IBlueRow r in ta)
{ // {
BlueTable retVal = new BlueTable (schema); // truly.addRow(r);
foreach (IBlueRow r in addedRows) // }
if (deletedRows.containsID ((int)r["id"]))
retVal.addRow (r);
return retVal;
}
// return truly;
public IBlueRowCollection getTrulyDeletedRows(params string[] idColumns)
{
BlueTable truly = new BlueTable(deletedRows.schema);
BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
CollectionHelpers.EqualDelegate ed = // }
new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed); // public virtual IBlueRowCollection getTrulyAddedRows ()
// {
foreach(IBlueRow r in td) // BlueTable retVal = new BlueTable (schema);
{ // foreach (IBlueRow r in addedRows)
truly.addRow(r); // if (!deletedRows.containsID ((int)r["id"]))
} // retVal.addRow (r);
// return retVal;
// }
return truly; // public virtual IBlueRowCollection getTrulyDeletedRows ()
} // {
// BlueTable retVal = new BlueTable (schema);
public IBlueRowCollection getTrulyChangedRows(params string[] idColumns) // foreach (IBlueRow r in deletedRows)
{ // if (!addedRows.containsID ((int)r["id"]))
// retVal.addRow (r);
BlueTable truly = new BlueTable(addedRows.schema); // return retVal;
BlueRowIdComparator c = new BlueRowIdComparator(idColumns); // }
CollectionHelpers.EqualDelegate ed = // public virtual IBlueRowCollection getTrulyChangedRows ()
new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); // {
// BlueTable retVal = new BlueTable (schema);
// foreach (IBlueRow r in addedRows)
// if (deletedRows.containsID ((int)r["id"]))
// retVal.addRow (r);
// return retVal;
// }
ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
ICollection tc = CollectionHelpers.difference(addedRows, ta, ed);
foreach(IBlueRow r in tc)
{
truly.addRow(r);
}
return truly;
}
public IBlueRowCollection getTrulyChangedOldRows(params string[] idColumns)
{
BlueTable truly = new BlueTable(deletedRows.schema);
BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
CollectionHelpers.EqualDelegate ed = // public IBlueRowCollection getTrulyDeletedRows(params string[] idColumns)
new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); // {
// BlueTable truly = new BlueTable(deletedRows.schema);
// BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed); // CollectionHelpers.EqualDelegate ed =
ICollection tco = CollectionHelpers.difference(deletedRows, td, ed); // new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
foreach(IBlueRow r in tco)
{
truly.addRow(r);
}
return truly; // ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
}
public TablePacket toTablePacket ()
{
return new TablePacket (BlueDataUtility.toDataTable (newRows), BlueDataUtility.toDataTable (addedRows),
BlueDataUtility.toDataTable (deletedRows));
}
public IBlueTableSchema schema // foreach(IBlueRow r in td)
{ // {
get // truly.addRow(r);
{ // }
IBlueTableSchema retVal = newRows.schema != null ? newRows.schema :
addedRows.schema != null ? addedRows.schema : deletedRows.schema;
return retVal;
}
}
/* // return truly;
status D total A total D this A This D' total A' Total // }
x 0 0 0 0 0 0
0 0 0 1 1 this+
0 0 1 0 1this
0 0 1 1 1 this 1 this+
x 0 1 0 0
error 0 1 0 1
0 1 1 0 1 total-
0 1 1 1 1 total-, 1 this +
x 1 0 0 0
1 0 0 1 1 this+
error 1 0 1 0
error 1 0 1 1
x 1 1 0 0
error 1 1 0 1
1 1 1 0 1 total -1 keep the old delete and remove the old add
1 1 1 1 1 total-, 1 this +
d' = dthis - (dthis intersect a) + d // keep the old delete
a' = a - (dthis intersect a) + athis // keep the new add
* */
public static DataPacket combineWithIds(ICollection packets, string[] idColumns)
{
if ( packets.Count == 0 )
{
return new DataPacket(
new BlueTable(new DataTable()),
new BlueTable(new DataTable()),
new BlueTable(new DataTable()));
}
DataPacket.BlueRowIdComparator c = new DataPacket.BlueRowIdComparator(idColumns); // public IBlueRowCollection getTrulyChangedRows(params string[] idColumns)
// {
CollectionHelpers.EqualDelegate ed =
new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
ArrayList added = new ArrayList(); // BlueTable truly = new BlueTable(addedRows.schema);
ArrayList deleted = new ArrayList(); // BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
BlueTable dt = null;
BlueTable at = null;
IBlueRowCollection nt = null;
foreach(DataPacket dp in packets) // CollectionHelpers.EqualDelegate ed =
{ // new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
ArrayList oldDeleted = deleted;
ArrayList oldAdded = added;
added = new ArrayList(); // ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
deleted = new ArrayList(); // ICollection tc = CollectionHelpers.difference(addedRows, ta, ed);
if ( dt == null )
{
dt = new BlueTable(dp.schema);
at = new BlueTable(dp.schema);
}
deleted.AddRange(oldDeleted);
deleted.AddRange( CollectionHelpers.difference(
CollectionHelpers.difference(dp.deletedRows, oldAdded, ed),
oldDeleted, ed) );
added.AddRange(dp.addedRows); // foreach(IBlueRow r in tc)
added.AddRange( CollectionHelpers.difference( // {
CollectionHelpers.difference(oldAdded, dp.deletedRows, ed), // truly.addRow(r);
added, ed) ); // }
nt = dp.newRows;
}
foreach ( IBlueRow dr in added )
{
at.addRow(dr);
}
foreach ( IBlueRow dr in deleted ) // return truly;
{ // }
dt.addRow(dr);
}
return new DataPacket(nt,at,dt); // public IBlueRowCollection getTrulyChangedOldRows(params string[] idColumns)
} // {
// BlueTable truly = new BlueTable(deletedRows.schema);
// BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
public DataPacket deepCopy () // CollectionHelpers.EqualDelegate ed =
{ // new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
return new DataPacket (new OrderedBlueTable (newRows),
new OrderedBlueTable (newRows),
new BlueTable (deletedRows));
}
public virtual DataPacket createImmutable () // ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
{ // ICollection tco = CollectionHelpers.difference(deletedRows, td, ed);
return
new DataPacket(
newRows is IImmutableRowCollection ? newRows : new ImmutableBlueTable(newRows),
addedRows,
deletedRows);
}
#region ICloneable Members
public object Clone() // foreach(IBlueRow r in tco)
{ // {
return MemberwiseClone(); // truly.addRow(r);
} // }
#endregion // return truly;
// }
public string toIncrementalIDString() // public TablePacket toTablePacket ()
{ // {
System.Text.StringBuilder sb = new System.Text.StringBuilder(schema.tableName); // return new TablePacket (BlueDataUtility.toDataTable (newRows), BlueDataUtility.toDataTable (addedRows),
sb.Append ("\ndeleted : "); // BlueDataUtility.toDataTable (deletedRows));
foreach (IBlueRow r in deletedRows)
{
sb.Append(r["id"]);
sb.Append(',');
}
sb.Append("\nadded : ");
foreach (IBlueRow r in addedRows)
{
sb.Append(r["id"]);
sb.Append(',');
}
sb.Append("\n");
return sb.ToString();
}
public override string ToString() // }
{
return String.Format("DataPacket, schema {0} n {1} d {2} a {3}", schema.tableName, newRows.Count, deletedRows.Count, addedRows.Count); // public IBlueTableSchema schema
} // {
} // get
} // {
// IBlueTableSchema retVal = newRows.schema != null ? newRows.schema :
// addedRows.schema != null ? addedRows.schema : deletedRows.schema;
// return retVal;
// }
// }
// /*
// status D total A total D this A This D' total A' Total
// x 0 0 0 0 0 0
// 0 0 0 1 1 this+
// 0 0 1 0 1this
// 0 0 1 1 1 this 1 this+
// x 0 1 0 0
// error 0 1 0 1
// 0 1 1 0 1 total-
// 0 1 1 1 1 total-, 1 this +
// x 1 0 0 0
// 1 0 0 1 1 this+
// error 1 0 1 0
// error 1 0 1 1
// x 1 1 0 0
// error 1 1 0 1
// 1 1 1 0 1 total -1 keep the old delete and remove the old add
// 1 1 1 1 1 total-, 1 this +
// d' = dthis - (dthis intersect a) + d // keep the old delete
// a' = a - (dthis intersect a) + athis // keep the new add
// * */
// public static DataPacket combineWithIds(ICollection packets, string[] idColumns)
// {
// if ( packets.Count == 0 )
// {
// return new DataPacket(
// new BlueTable(new DataTable()),
// new BlueTable(new DataTable()),
// new BlueTable(new DataTable()));
// }
// DataPacket.BlueRowIdComparator c = new DataPacket.BlueRowIdComparator(idColumns);
// CollectionHelpers.EqualDelegate ed =
// new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
// ArrayList added = new ArrayList();
// ArrayList deleted = new ArrayList();
// BlueTable dt = null;
// BlueTable at = null;
// IBlueRowCollection nt = null;
// foreach(DataPacket dp in packets)
// {
// ArrayList oldDeleted = deleted;
// ArrayList oldAdded = added;
// added = new ArrayList();
// deleted = new ArrayList();
// if ( dt == null )
// {
// dt = new BlueTable(dp.schema);
// at = new BlueTable(dp.schema);
// }
// deleted.AddRange(oldDeleted);
// deleted.AddRange( CollectionHelpers.difference(
// CollectionHelpers.difference(dp.deletedRows, oldAdded, ed),
// oldDeleted, ed) );
// added.AddRange(dp.addedRows);
// added.AddRange( CollectionHelpers.difference(
// CollectionHelpers.difference(oldAdded, dp.deletedRows, ed),
// added, ed) );
// nt = dp.newRows;
// }
// foreach ( IBlueRow dr in added )
// {
// at.addRow(dr);
// }
// foreach ( IBlueRow dr in deleted )
// {
// dt.addRow(dr);
// }
// return new DataPacket(nt,at,dt);
// }
// public DataPacket deepCopy ()
// {
// return new DataPacket (new OrderedBlueTable (newRows),
// new OrderedBlueTable (newRows),
// new BlueTable (deletedRows));
// }
// public virtual DataPacket createImmutable ()
// {
// return
// new DataPacket(
// newRows is IImmutableRowCollection ? newRows : new ImmutableBlueTable(newRows),
// addedRows,
// deletedRows);
// }
// #region ICloneable Members
// public object Clone()
// {
// return MemberwiseClone();
// }
// #endregion
// public string toIncrementalIDString()
// {
// System.Text.StringBuilder sb = new System.Text.StringBuilder(schema.tableName);
// sb.Append ("\ndeleted : ");
// foreach (IBlueRow r in deletedRows)
// {
// sb.Append(r["id"]);
// sb.Append(',');
// }
// sb.Append("\nadded : ");
// foreach (IBlueRow r in addedRows)
// {
// sb.Append(r["id"]);
// sb.Append(',');
// }
// sb.Append("\n");
// return sb.ToString();
// }
// public override string ToString()
// {
// return String.Format("DataPacket, schema {0} n {1} d {2} a {3}", schema.tableName, newRows.Count, deletedRows.Count, addedRows.Count);
// }
// }
// }
@@ -2,171 +2,151 @@ using System;
using System.Collections; using System.Collections;
using System.IO; using System.IO;
namespace BCG namespace BCG {
{ /// <summary>
/// <summary> /// The only instance of IPubTableBroker. It handles the publishing of one tag, and holds a specialized serializer
/// The only instance of IPubTableBroker. It handles the publishing of one tag, and holds a specialized serializer /// that it initializes according to the table, to optimize serialiazation, hence bw and performance
/// that it initializes according to the table, to optimize serialiazation, hence bw and performance /// This class is held (as an interface) by the PubTableBroker, which guarantees the sanity of calls,
/// This class is held (as an interface) by the PubTableBroker, which guarantees the sanity of calls, /// ie the first call will always be create.
/// ie the first call will always be create. /// </summary>
/// </summary> public class PubTablePublisher : IPubTablePublisher {
public class PubTablePublisher : IPubTablePublisher private readonly string _tag;
{
protected IBlueDataSerializer mySerializer; protected IBlueDataSerializer mySerializer;
protected PublisherCore myPubCore; protected PublisherCore myPubCore;
public PubTablePublisher (PublisherCore publisherCore) public PubTablePublisher(string tag, PublisherCore publisherCore) {
{ _tag = tag;
bts.Log.Gen.info($"PubTablePublisher created for tag: {tag}");
myPubCore = publisherCore; myPubCore = publisherCore;
mySerializer = new BlueDataSerializationController (); mySerializer = new BlueDataSerializationController();
} }
#region IPubTablePublisher Members #region IPubTablePublisher Members
public virtual void addRow (IBlueRow row) public virtual void addRow(IBlueRow row) {
{
byte[] data; byte[] data;
mySerializer.serialize (StreamOpCode.add, mySerializer.serialize(StreamOpCode.add,
myGetRowCollectionArray (row), myGetRowCollectionArray(row),
out data); out data);
myPublish (data, row.schema.tableName); myPublish(data, row.schema.tableName);
} }
public virtual void deleteRow (IBlueRow row) public virtual void deleteRow(IBlueRow row) {
{
byte[] data; byte[] data;
mySerializer.serialize (StreamOpCode.remove, mySerializer.serialize(StreamOpCode.remove,
myGetRowCollectionArray (row), myGetRowCollectionArray(row),
out data); out data);
myPublish (data, row.schema.tableName); myPublish(data, row.schema.tableName);
} }
public virtual void changeRow(IBlueRow row) public virtual void changeRow(IBlueRow row) {
{
byte[] data; byte[] data;
mySerializer.serialize (StreamOpCode.change, mySerializer.serialize(StreamOpCode.change,
myGetRowCollectionArray (row), myGetRowCollectionArray(row),
out data); out data);
myPublish (data, row.schema.tableName); myPublish(data, row.schema.tableName);
} }
public virtual void createTable (IBlueRowCollection rows) public virtual void createTable(IBlueRowCollection rows) {
{
byte[] data; byte[] data;
mySerializer.serialize (StreamOpCode.create, mySerializer.serialize(StreamOpCode.create,
new IBlueRowCollection [] {rows}, new IBlueRowCollection[] { rows },
out data); out data);
myPublish (data, rows.schema.tableName); bts.Log.Gen.info($"PubTablePublisher.createTable tag: {_tag} rows: {rows.Count} data: {data.Length}");
myPublish(data, rows.schema.tableName);
} }
public virtual void batchRemoveAdd (IBlueRowCollection delRows, IBlueRowCollection addRows) public virtual void batchRemoveAdd(IBlueRowCollection delRows, IBlueRowCollection addRows) {
{
byte[] data; byte[] data;
if (myIsRealChange (delRows, addRows)) if (myIsRealChange(delRows, addRows))
batchChange (addRows); batchChange(addRows);
else else {
{ mySerializer.serialize(StreamOpCode.multipleRemoveAdd,
mySerializer.serialize (StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { delRows, addRows },
new IBlueRowCollection []{delRows, addRows},
out data); out data);
myPublish (data, delRows.schema.tableName); myPublish(data, delRows.schema.tableName);
} }
} }
public virtual void batchChange (IBlueRowCollection rows) public virtual void batchChange(IBlueRowCollection rows) {
{
byte[] data; byte[] data;
mySerializer.serialize (StreamOpCode.multipleChange, mySerializer.serialize(StreamOpCode.multipleChange,
new IBlueRowCollection [] {rows}, new IBlueRowCollection[] { rows },
out data); out data);
myPublish (data, rows.schema.tableName); myPublish(data, rows.schema.tableName);
} }
#endregion #endregion
#region private methods #region private methods
private void myPublish (byte[] data, string tableName) private void myPublish(byte[] data, string tableName) {
{ bts.Log.Gen.info($"PubTablePublisher.myPublish table: {tableName} data: {data.Length}");
if (data.Length > 0) if (data.Length > 0)
myPubCore.publish ("DataStreamTag", myPubCore.publish("DataStreamTag",
PubStreamPrefixer.prefix (tableName), PubStreamPrefixer.prefix(tableName),
new MemoryStream (data, 0, data.Length, false, true)); new MemoryStream(data, 0, data.Length, false, true));
} }
private IBlueRowCollection[] myGetRowCollectionArray (IBlueRow r) private IBlueRowCollection[] myGetRowCollectionArray(IBlueRow r) {
{ return new IBlueRowCollection[]{
return new IBlueRowCollection []{
BlueDataUtility.toBlueRowCollection (new IBlueRow[] {r}) BlueDataUtility.toBlueRowCollection (new IBlueRow[] {r})
}; };
} }
private bool myIsRealChange (IBlueRowCollection delRows, IBlueRowCollection addRows) private bool myIsRealChange(IBlueRowCollection delRows, IBlueRowCollection addRows) {
{ if (delRows.Count == addRows.Count) {
if (delRows.Count == addRows.Count)
{
foreach (IBlueRow dRow in delRows) foreach (IBlueRow dRow in delRows)
if (!addRows.containsID ((int)dRow["id"])) if (!addRows.containsID((int)dRow["id"]))
return false; return false;
return true; return true;
} } else
else
return false; return false;
} }
#endregion #endregion
} }
public class IndependentTablePublisher : PubTablePublisher public class IndependentTablePublisher : PubTablePublisher {
{
bool myIsCreated = false; bool myIsCreated = false;
public IndependentTablePublisher (PublisherCore pubCore) : base (pubCore) public IndependentTablePublisher(string tag, PublisherCore pubCore) : base(tag, pubCore) { }
{}
public override void addRow (IBlueRow row) public override void addRow(IBlueRow row) {
{ myCheckCreate(row.schema);
myCheckCreate (row.schema); base.addRow(row);
base.addRow (row);
} }
public override void batchChange (IBlueRowCollection rows) public override void batchChange(IBlueRowCollection rows) {
{ myCheckCreate(rows.schema);
myCheckCreate (rows.schema); base.batchChange(rows);
base.batchChange (rows);
} }
public override void batchRemoveAdd (IBlueRowCollection delRows, IBlueRowCollection addRows) public override void batchRemoveAdd(IBlueRowCollection delRows, IBlueRowCollection addRows) {
{ myCheckCreate(delRows.schema);
myCheckCreate (delRows.schema); base.batchRemoveAdd(delRows, addRows);
base.batchRemoveAdd (delRows, addRows);
} }
public override void changeRow (IBlueRow row) public override void changeRow(IBlueRow row) {
{ myCheckCreate(row.schema);
myCheckCreate (row.schema); base.changeRow(row);
base.changeRow (row);
} }
public override void deleteRow (IBlueRow row) public override void deleteRow(IBlueRow row) {
{ myCheckCreate(row.schema);
myCheckCreate (row.schema); base.deleteRow(row);
base.deleteRow (row);
} }
public override void createTable(IBlueRowCollection rows) public override void createTable(IBlueRowCollection rows) {
{
myIsCreated = true; myIsCreated = true;
base.createTable (rows); base.createTable(rows);
} }
private void myCheckCreate (IBlueTableSchema schema) private void myCheckCreate(IBlueTableSchema schema) {
{ if (!myIsCreated) {
if (!myIsCreated)
{
//so that the initiazliation work is done on the serializer //so that the initiazliation work is done on the serializer
byte[] placeHolder; byte[] placeHolder;
mySerializer.serialize (StreamOpCode.create, new IBlueRowCollection [] {new BlueTable (schema)}, out placeHolder); mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { new BlueTable(schema) }, out placeHolder);
myIsCreated = true; myIsCreated = true;
} }
} }
@@ -1,18 +1,16 @@
using System; using System;
namespace BCG namespace BCG {
{ /// <summary>
/// <summary> /// Deals with the publication of one tag. Gets the specific implementaion of IBlueDataSerializer according to
/// Deals with the publication of one tag. Gets the specific implementaion of IBlueDataSerializer according to /// the table name, from the config, and wraps around it. The external code that will deal with user calls for
/// the table name, from the config, and wraps around it. The external code that will deal with user calls for /// row operations (PubTablePublisher) will only deal with this class, hence will be agnostic of the specalized
/// row operations (PubTablePublisher) will only deal with this class, hence will be agnostic of the specalized /// serialization.
/// serialization. /// </summary>
/// </summary> public class BlueDataSerializationController : IBlueDataSerializer {
public class BlueDataSerializationController : IBlueDataSerializer
{
private IBlueDataSerializer mySerializer; private IBlueDataSerializer mySerializer;
public BlueDataSerializationController () {} public BlueDataSerializationController() { }
#region IBlueDataSerializer Members #region IBlueDataSerializer Members
@@ -23,43 +21,37 @@ namespace BCG
/// <param name="op"></param> /// <param name="op"></param>
/// <param name="rows"></param> /// <param name="rows"></param>
/// <param name="serializedData"></param> /// <param name="serializedData"></param>
public void serialize (byte op, IBlueRowCollection[] rows, out byte[] serializedData) public void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData) {
{ if (op == StreamOpCode.create) {
if (op == StreamOpCode.create)
{
if (mySerializer == null) if (mySerializer == null)
mySerializer = myGetSerializerForTable(rows[0].schema); mySerializer = myGetSerializerForTable(rows[0].schema);
mySerializer.serialize (op, rows, out serializedData); mySerializer.serialize(op, rows, out serializedData);
} } else {
else if (mySerializer != null) {
{ mySerializer.serialize(op, rows, out serializedData);
if (mySerializer != null) } else {
{ serializedData = new byte[] { };
mySerializer.serialize (op, rows, out serializedData);
}
else
{
serializedData = new byte[] {};
} }
} }
} }
#endregion #endregion
private IBlueDataSerializer myGetSerializerForTable(IBlueTableSchema theSchema) private IBlueDataSerializer myGetSerializerForTable(IBlueTableSchema theSchema) {
{ // TODOds
return new UniversalSerializer(theSchema);
string serializerName = DataStreamGatewaySettings.instance.getSerializer(theSchema.tableName); string serializerName = DataStreamGatewaySettings.instance.getSerializer(theSchema.tableName);
object[] serializerArgs = DataStreamGatewaySettings.instance.getSerializerArgs(theSchema.tableName); object[] serializerArgs = DataStreamGatewaySettings.instance.getSerializerArgs(theSchema.tableName);
if(serializerArgs != null) if (serializerArgs != null)
return Activator.CreateInstance(Type.GetType(serializerName), theSchema, serializerArgs) as IBlueDataSerializer; return Activator.CreateInstance(Type.GetType(serializerName), theSchema, serializerArgs) as IBlueDataSerializer;
else else
return Activator.CreateInstance(Type.GetType(serializerName), theSchema) as IBlueDataSerializer; return Activator.CreateInstance(Type.GetType(serializerName), theSchema) as IBlueDataSerializer;
} }
} }
} }
@@ -8,7 +8,7 @@ namespace BCG
public class UniversalSerializer : BlueDataSerializer public class UniversalSerializer : BlueDataSerializer
{ {
public UniversalSerializer (IBlueTableSchema schema) public UniversalSerializer (IBlueTableSchema schema)
: base ("BCG.PubsubUniversalDeserializer, DataStreamGateway", "BCG.ClientUniversalDeserializer") : base ("BCG.PubsubUniversalDeserializer, DataStreamCore", "BCG.ClientUniversalDeserializer")
{ {
myRowSerializer = new UniversalRowSerializer (schema); myRowSerializer = new UniversalRowSerializer (schema);
} }
@@ -7,10 +7,11 @@ using System.Text;
namespace BCG namespace BCG
{ {
/// <summary> /// <summary>
/// Base class for all deserializers. /// Base class for all pubsubs deserializers.
/// </summary> /// </summary>
public abstract class PubsubDeserializer : IPubsubDeserializer public abstract class PubsubDeserializer : IPubsubDeserializer
{ {
// this is where the stream state is stored in pubsub?
protected IDictionary myState = new Hashtable (); protected IDictionary myState = new Hashtable ();
protected IBlueTableSchema mySchema; protected IBlueTableSchema mySchema;
protected string myClientDeserializerName; //for the getState method protected string myClientDeserializerName; //for the getState method
@@ -35,6 +36,7 @@ namespace BCG
} }
catch (EndOfStreamException) catch (EndOfStreamException)
{ {
// TODO, logically break loop
break; break;
} }
if (op == StreamOpCode.create) if (op == StreamOpCode.create)
@@ -2,24 +2,22 @@ using System;
using System.Collections; using System.Collections;
using System.IO; using System.IO;
namespace BCG namespace BCG {
{ public class PubsubUniversalDeserializer : PubsubDeserializer {
public class PubsubUniversalDeserializer : PubsubDeserializer public PubsubUniversalDeserializer() {
{ bts.Log.Gen.info("PubsubUniversalDeserializer.cons");
public PubsubUniversalDeserializer() {} }
protected override IDictionary myGetRows (BinaryReader br, byte op, int noOfRows) protected override IDictionary myGetRows(BinaryReader br, byte op, int noOfRows) {
{ myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer(mySchema) : myDeserializer;
myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer (mySchema) : myDeserializer; Hashtable retVal = new Hashtable();
Hashtable retVal = new Hashtable (); for (int i = 0; i < noOfRows; i++) {
for (int i = 0; i < noOfRows; i++)
{
int id; object row; int id; object row;
myDeserializer.deserializeRow (myState, br, op, out id, out row); myDeserializer.deserializeRow(myState, br, op, out id, out row);
retVal.Add (id, row); retVal.Add(id, row);
} }
return retVal; return retVal;
} }
} }
} }
@@ -1,54 +1,46 @@
using System; using System;
using System.IO; using System.IO;
using System.Collections; using System.Collections.Generic;
using BCG.Utilities; using BCG.Utilities;
namespace BCG namespace BCG {
{
/// <summary> /// <summary>
/// Keeps a collection of TagStateBanks to hold the state of each tag /// Keeps a collection of TagStateBanks to hold the state of each tag
/// </summary> /// </summary>
public class PubsubServerTagHandler : IPubSubServerTagHandler public class PubsubServerTagHandler : IPubSubServerTagHandler {
{ private IDictionary<string, TagStateBank> _tag_state_map = new Dictionary<string, TagStateBank>();
private IDictionary myTagStates = new Hashtable ();
private bool myPublishStatistics = //DataStreamGatewaySettings.instance.enablePubsubStats; private bool myPublishStatistics = //DataStreamGatewaySettings.instance.enablePubsubStats;
false; false;
public PubsubServerTagHandler() {}
#region IPubSubServerTagHandler Members #region IPubSubServerTagHandler Members
public byte[] getContent (string tag) public byte[] getContent(string tag) {
{ TagStateBank tag_state_bank;
TagStateBank stateBank = myTagStates[tag] as TagStateBank; if (_tag_state_map.TryGetValue(tag, out tag_state_bank)) {
if (stateBank != null) return tag_state_bank.getConent();
{ } else {
return stateBank.getConent (); bts.Log.Gen.warn("Information requested for tag: {tag}, it does not exist now");
} return new byte[] { };
else
{
// BlueEventLog.logEvent (String.Format ("Information requested for tag {0} which does not exist now", tag), BlueEventLevel.Warning);
return new byte[] {};
} }
} }
public void update (string tag, Stream[] messageBodies) public void update(string tag, Stream[] messageBodies) {
{ bts.Log.Gen.info($"PubsubServerTagHandler.update tag: {tag} streams: {messageBodies.Length}");
if (myPublishStatistics) if (myPublishStatistics) {
{
StatisticsLogger.instance.log(tag, messageBodies); StatisticsLogger.instance.log(tag, messageBodies);
} }
if (!myTagStates.Contains (tag)) TagStateBank tag_state_bank;
{ if (!_tag_state_map.TryGetValue(tag, out tag_state_bank)) {
// BlueEventLog.logEvent (String.Format ("Tag {0} received for the first time and its state created", tag), BlueEventLevel.Information); bts.Log.Gen.info("tag: {tag} received for the first time creating TagStateBank for it now");
myTagStates.Add (tag, new TagStateBank ()); tag_state_bank = new TagStateBank();
_tag_state_map.Add(tag, tag_state_bank);
} }
(myTagStates[tag] as TagStateBank).update (messageBodies); tag_state_bank.update(messageBodies);
} }
public void doWork() {} public void doWork() { }
public string[] supportedTagTypes {get {return new string [] {"DataStreamTag"};}} public string[] supportedTagTypes { get { return new string[] { "DataStreamTag" }; } }
public void register(InternalPublishDelegate callback) { } public void register(InternalPublishDelegate callback) { }
@@ -1,26 +1,26 @@
using System; using System;
using System.Collections; using System.Collections.Generic;
using System.IO; using System.IO;
namespace BCG namespace BCG {
{ public class TagStateBank {
public class TagStateBank private PubsubDeserializationController myDeserializationController = new PubsubDeserializationController();
{
//private IDictionary myState = new Hashtable ();
private PubsubDeserializationController myDeserializationController = new PubsubDeserializationController ();
public TagStateBank () {}
public byte[] getConent () public TagStateBank() { }
{
return myDeserializationController.getState (); public TagStateBank(string tag) {
bts.Log.Gen.info("TagStateBank created for tag: {tag}");
} }
public void update (Stream[] streams) public byte[] getConent() {
{ return myDeserializationController.getState();
}
public void update(Stream[] streams) {
bts.Log.Gen.info($"TagStateBank.update streams: {streams.Length}");
byte[] ops; byte[] ops;
Array rowCollection; Array rowCollection;
myDeserializationController.deserialize (streams, out ops, out rowCollection); myDeserializationController.deserialize(streams, out ops, out rowCollection);
} }
} }
} }
@@ -9,7 +9,7 @@ namespace BCG
public class CompressionSerializer : IBlueDataSerializer public class CompressionSerializer : IBlueDataSerializer
{ {
private IBlueRowSerializer myRowSerializer; private IBlueRowSerializer myRowSerializer;
private string myPubsubDeserializerName = "BCG.CompressionPubsubDeserializer, DataStreamGateway", myClientDeserializerName = "BCG.CompressionClientDeserializer"; private string myPubsubDeserializerName = "BCG.CompressionPubsubDeserializer, DataStreamCore", myClientDeserializerName = "BCG.CompressionClientDeserializer";
public CompressionSerializer (IBlueTableSchema schema) public CompressionSerializer (IBlueTableSchema schema)
{ {
@@ -0,0 +1,236 @@
// using System;
// using System.IO;
// using NUnit.Framework;
// namespace BCG
// {
// /// <summary>
// /// Tests the deserialization happening within the pubsub.
// /// </summary>
// [TestGroup]
// public class CompressionSerializationTest
// {
// protected IBlueTableSchema mySchema;
// protected BlueTable myTable;
// protected IBlueDataSerializer mySer;
// protected IPubsubDeserializer myPubsubDeser;
// protected IClientDeserializer myClientDeser;
// [Test]
// public static void test_compression()
// {
// int n_len = 10000000;
// for (int t = 0; t < 10; t++)
// {
// Random rnd = new Random(DateTime.Now.Millisecond);
// byte[] data = new byte[n_len];
// rnd.NextBytes(data);
// BCG.Utilities.General.HighPerformanceTimer timer = new Utilities.General.HighPerformanceTimer();
// timer.start();
// byte[] compr = UtilityMethods.compress(new MemoryStream(data));
// timer.stop();
// Console.WriteLine("Time to compress {0} bytes {1} msecs ratio {2}", n_len, timer.lastIntervalTime, (1.0 * data.Length)/(1.0 * compr.Length));
// timer.start();
// Stream decompr_str = UtilityMethods.decompress(compr);
// timer.stop();
// Console.WriteLine("Time to de-compress {0} bytes {1} msecs", n_len, timer.lastIntervalTime);
// decompr_str.Position = 0;
// Assert.AreEqual(data.Length, (int)decompr_str.Length);
// for (int i = 0; i < data.Length; i++)
// Assert.AreEqual(data[i], decompr_str.ReadByte());
// }
// }
// public CompressionSerializationTest()
// {
// mySchema = new BlueTableSchema(myGetTableName(),
// new ColumnInfo("id", 0, typeof(int)),
// new ColumnInfo("i1", 1, typeof(int)),
// new ColumnInfo("d1", 2, typeof(double)),
// new ColumnInfo("dt", 3, typeof(DateTime)),
// new ColumnInfo("x", 4, typeof(byte))
// );
// mySer = new CompressionSerializer(mySchema);
// myClientDeser = new ClientDeserializationController();
// myPubsubDeser = new PubsubDeserializationController();
// }
// protected virtual string myGetTableName()
// {
// return "testUniversal";
// }
// /// <summary>
// /// Tests the entire cycle. This is done by 'publishing' data to the serializationController (ie a series of
// /// adds, deletions, changes etc) and getting the bytes and feeding it into a pubsubDeseriazlier and a clientDeserializer.
// /// At every step the data is tested for consistency. At the end the getState is used to get all the data, and its passed
// /// through the clientDeserializer to get all the rows.
// /// </summary>
// [Test]
// public void testEntireProcess()
// {
// myTable = new BlueTable(mySchema);
// byte[] data;
// int id;
// //create
// for (id = 0; id < 100; id++)
// myTable.addRow(myGetRow(id));
// byte[] ops;
// Array rows;
// IBlueRowCollection del, add;
// IBlueRowCollection[] receivedData;
// IBlueRow r;
// //create
// mySer.serialize(StreamOpCode.create, new IBlueRowCollection[] { myTable }, out data);
// myWrite(StreamOpCode.create, data);
// myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// Assert.AreEqual(1, ops.Length);
// Assert.AreEqual(StreamOpCode.create, ops[0]);
// Assert.AreEqual(1, rows.Length);
// receivedData = rows.GetValue(0) as IBlueRowCollection[];
// Assert.AreEqual(1, receivedData.Length);
// BlueDataUtility.computeDifferences(receivedData[0], myTable, out del, out add);
// Assert.AreEqual(0, del.Count);
// Assert.AreEqual(0, add.Count);
// //single add
// r = myGetRow(id);
// BlueTable temp = new BlueTable(mySchema);
// myTable.addRow(r);
// temp.addRow(r);
// mySer.serialize(StreamOpCode.add, new IBlueRowCollection[] { temp }, out data);
// myWrite(StreamOpCode.add, data);
// myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// Assert.AreEqual(1, ops.Length);
// Assert.AreEqual(StreamOpCode.add, ops[0]);
// Assert.AreEqual(1, rows.Length);
// receivedData = rows.GetValue(0) as IBlueRowCollection[];
// Assert.AreEqual(1, receivedData.Length);
// BlueDataUtility.computeDifferences(receivedData[0], temp, out del, out add);
// Assert.AreEqual(0, del.Count);
// Assert.AreEqual(0, add.Count);
// //single delete
// r = myTable[0]; //delete row with id 0
// temp = new BlueTable(mySchema);
// myTable.deleteRow(0);
// temp.addRow(r);
// mySer.serialize(StreamOpCode.remove, new IBlueRowCollection[] { temp }, out data);
// myWrite(StreamOpCode.remove, data);
// myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// Assert.AreEqual(1, ops.Length);
// Assert.AreEqual(StreamOpCode.remove, ops[0]);
// Assert.AreEqual(1, rows.Length);
// receivedData = rows.GetValue(0) as IBlueRowCollection[];
// Assert.AreEqual(1, receivedData.Length);
// BlueDataUtility.computeDifferences(receivedData[0], temp, out del, out add);
// Assert.AreEqual(0, del.Count);
// Assert.AreEqual(0, add.Count);
// //batch deletes and adds
// //0 was removed earlier, so lets delete from 1 to 24 first, and then add 101 to 124
// BlueTable delRows = new BlueTable(mySchema), addRows = new BlueTable(mySchema);
// for (int i = 1; i < 25; i++)
// {
// delRows.addRow(myTable[i]);
// myTable.deleteRow(i);
// }
// for (id = 101; id < 125; id++)
// {
// r = myGetRow(id);
// addRows.addRow(r);
// myTable.addRow(r);
// }
// for (int i = 25; i < 125; i++)
// Assert.AreEqual(true, myTable.containsID(i));
// Assert.AreEqual(100, myTable.Count);
// mySer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { delRows, addRows }, out data);
// myWrite(StreamOpCode.multipleRemoveAdd, data);
// myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// Assert.AreEqual(1, ops.Length);
// Assert.AreEqual(StreamOpCode.multipleRemoveAdd, ops[0]);
// Assert.AreEqual(1, rows.Length);
// receivedData = rows.GetValue(0) as IBlueRowCollection[];
// Assert.AreEqual(2, receivedData.Length);
// BlueDataUtility.computeDifferences(receivedData[0], delRows, out del, out add);
// Assert.AreEqual(0, del.Count);
// Assert.AreEqual(0, add.Count);
// BlueDataUtility.computeDifferences(receivedData[1], addRows, out del, out add);
// Assert.AreEqual(0, del.Count);
// Assert.AreEqual(0, add.Count);
// //pure changes only
// //change the first 25, 25-49
// BlueTable changedRows = new BlueTable(mySchema);
// for (int i = 25; i < 50; i++)
// {
// MutableBlueRow mr = new MutableBlueRow(myTable[i]);
// mr.setValue(4, Byte.MaxValue);
// myTable.changeRow(mr);
// changedRows.addRow(mr);
// }
// mySer.serialize(StreamOpCode.multipleChange, new IBlueRowCollection[] { changedRows }, out data);
// myWrite(StreamOpCode.multipleChange, data);
// myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// Assert.AreEqual(1, ops.Length);
// Assert.AreEqual(StreamOpCode.multipleChange, ops[0]);
// Assert.AreEqual(1, rows.Length);
// receivedData = rows.GetValue(0) as IBlueRowCollection[];
// Assert.AreEqual(1, receivedData.Length);
// BlueDataUtility.computeDifferences(receivedData[0], changedRows, out del, out add);
// Assert.AreEqual(0, del.Count);
// Assert.AreEqual(0, add.Count);
// //final test
// data = myPubsubDeser.getState();
// new ClientDeserializationController().deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// Assert.AreEqual(1, ops.Length);
// Assert.AreEqual(StreamOpCode.create, ops[0]);
// Assert.AreEqual(1, rows.Length);
// receivedData = rows.GetValue(0) as IBlueRowCollection[];
// Assert.AreEqual(1, receivedData.Length);
// BlueDataUtility.computeDifferences(receivedData[0], myTable, out del, out add);
// Assert.AreEqual(0, del.Count);
// Assert.AreEqual(0, add.Count);
// }
// private IBlueRow myGetRow(int id)
// {
// return new BlueRow(mySchema, // the day pp fixed this test.
// id, id + 10, id * 3.0, new DateTime(2008, 6, 4, 0, 0, 0, DateTimeKind.Unspecified), Convert.ToByte(id));
// }
// private void myWrite(byte op, byte[] data)
// {
// string s = "";
// if (op == StreamOpCode.add)
// s = "add one row";
// else if (op == StreamOpCode.change)
// s = "change one row";
// else if (op == StreamOpCode.create)
// s = "create";
// else if (op == StreamOpCode.multipleChange)
// s = "multiple change";
// else if (op == StreamOpCode.multipleRemoveAdd)
// s = "multiple add remove";
// else if (op == StreamOpCode.remove)
// s = "remove one row";
// Console.WriteLine("op = {0}, size = {1}", s, data.Length);
// }
// }
// }
@@ -0,0 +1,100 @@
// using System;
// using System.IO;
// using System.Collections;
// using NUnit.Framework;
// namespace BCG
// {
// [TestGroup]
// public class FixedChangingRowSerializationTest
// {
// private IBlueTableSchema mySchema = new BlueTableSchema ("Test",
// new ColumnInfo ("i", 0, typeof(int)),
// new ColumnInfo ("b", 1, typeof(bool)),
// new ColumnInfo ("d", 2, typeof(double)),
// new ColumnInfo ("id", 3, typeof(int)),
// new ColumnInfo ("dt", 4, typeof(DateTime)),
// new ColumnInfo ("bt", 5, typeof(byte)));
// private BlueTable myState;
// private Random myRnd = new Random (DateTime.Now.Millisecond);
// private IBlueRowSerializer mySerializer;
// private IClientRowDeserializer myRowDeserializer;
// public FixedChangingRowSerializationTest ()
// {
// myState = new BlueTable (mySchema);
// mySerializer = new FixedChangingRowSerializer (mySchema, "bt");
// myRowDeserializer = new ClientFixedChangingRowDeserializer (mySchema, new string[] {"bt"});
// }
// [Test]
// public void addTest ()
// {
// for (int i = 0; i < 1000; i++)
// {
// BinaryWriter bw = new BinaryWriter (new MemoryStream ());
// IBlueRow r = myCreateRow (i);
// mySerializer.serializeRow (r, StreamOpCode.add, bw);
// BinaryReader br = new BinaryReader (new MemoryStream ((bw.BaseStream as MemoryStream).ToArray ()));
// Assert.AreEqual (r["i"], br.ReadInt32());
// Assert.AreEqual (r["b"], br.ReadBoolean ());
// Assert.AreEqual (r["d"], br.ReadDouble ());
// Assert.AreEqual (r["id"], br.ReadInt32 ());
// Assert.AreEqual (((DateTime)r["dt"]).Ticks, br.ReadInt64());
// Assert.AreEqual (r["bt"], br.ReadByte ());
// }
// }
// [Test]
// public void removeTest ()
// {
// for (int i = 0; i < 1000; i++)
// {
// BinaryWriter bw = new BinaryWriter (new MemoryStream ());
// IBlueRow r = myCreateRow (i);
// mySerializer.serializeRow (r, StreamOpCode.remove, bw);
// BinaryReader br = new BinaryReader (new MemoryStream ((bw.BaseStream as MemoryStream).ToArray ()));
// Assert.AreEqual (r["id"], br.ReadInt32());
// }
// }
// [Test]
// public void changeTest ()
// {
// for (int i = 0; i < 1000; i++)
// {
// BinaryWriter bw = new BinaryWriter (new MemoryStream ());
// IBlueRow r = myCreateRow (i);
// mySerializer.serializeRow (r, StreamOpCode.change, bw);
// BinaryReader br = new BinaryReader (new MemoryStream ((bw.BaseStream as MemoryStream).ToArray ()));
// Assert.AreEqual (r["id"], br.ReadInt32());
// Assert.AreEqual (r["bt"], br.ReadByte());
// bool isEnd = false;
// try
// {
// br.ReadByte ();
// }
// catch (EndOfStreamException)
// {
// isEnd = true;
// }
// Assert.That (isEnd);
// }
// }
// private IBlueRow myCreateRow (int id)
// {
// return new BlueRow (mySchema,
// myRnd.Next (1000),
// myRnd.Next (1000) % 2 == 0,
// myRnd.NextDouble (),
// id,
// DateTime.Now,
// Convert.ToByte (10));
// }
// }
// }
@@ -0,0 +1,40 @@
// using System;
// namespace BCG
// {
// public class TestFixedChangingSerializer : FixedChangingSerializer
// {
// public TestFixedChangingSerializer (IBlueTableSchema schema)
// : base (schema,
// new string[] {"x"},
// "BCG.PubsubTestFixedChangingDeserializer",
// "BCG.ClientTestFixedChangingDeserializer"
// )
// {}
// }
// public class PubsubTestFixedChangingDeserializer : PubsubFixedChangingDeserializer
// {
// public PubsubTestFixedChangingDeserializer () : base (new string[] {"x"})
// {}
// }
// public class ClientTestFixedChangingDeserializer : ClientFixedChangingDeserializer
// {
// public ClientTestFixedChangingDeserializer () : base (new string[] {"x"})
// {}
// }
// public class FixedChangingSerializationTest : UniversalSerializationTest
// {
// public FixedChangingSerializationTest () {}
// protected override string myGetTableName()
// {
// return "testFixedChanging";
// }
// }
// }
@@ -0,0 +1,73 @@
// using System;
// using System.Collections;
// using System.IO;
// using NUnit.Framework;
// using BCG.Utilities.General;
// namespace BCG
// {
// [TestFixture]
// public class MarketOptionTheoSerializationTest
// {
// private IBlueTableSchema mySchema = new BlueTableSchema ("marketOptionTheo",
// new ColumnInfo ("arrayIndex", 0, typeof(byte)),
// new ColumnInfo ("cell", 1, typeof(byte)),
// new ColumnInfo ("expiration", 2, typeof(DateTime)),
// new ColumnInfo ("id", 3, typeof(int)),
// new ColumnInfo ("value", 4, typeof(double))
// );
// public MarketOptionTheoSerializationTest () {}
// [Test]
// public void serializationPerformanceTest ()
// {
// MarketOptionTheoSerializer ser = new MarketOptionTheoSerializer (mySchema);
// IClientDeserializer deser = new MarketOptionTheoClientDeserializer ();
// BlueTable bt = new BlueTable (mySchema);
// int count = 3000;
// byte[] ops;
// Array rows;
// HighPerformanceTimer timer = new HighPerformanceTimer ();
// byte[] serData = null;
// for (int i = 0; i < count; i++)
// bt.addRow (myGetRow (i));
// ///serialization
// //create
// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out serData);
// deser.deserialize (new Stream [] {new MemoryStream (serData)}, out ops, out rows);
// ArrayList times = new ArrayList ();
// for (int i = 0; i < 25; i++)
// {
// timer.start ();
// ser.serialize (StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] {bt, bt}, out serData);
// timer.stop ();
// times.Add (timer.lastIntervalTime);
// Console.WriteLine (String.Format ("serialization of {0} rows, time = {1}, size = {2}", count, timer.lastIntervalTime, serData.Length));
// }
// times.Sort ();
// Console.WriteLine ("Median time = {0}", times[times.Count/2]);
// //deserialization
// times.Clear ();
// for (int i = 0; i < 25; i++)
// {
// timer.start ();
// deser.deserialize (new Stream [] {new MemoryStream (serData)}, out ops, out rows);
// timer.stop();
// times.Add (timer.lastIntervalTime);
// Console.WriteLine (String.Format ("deserialization of {0} rows, time = {1}", count, timer.lastIntervalTime));
// }
// times.Sort ();
// Console.WriteLine ("Median time = {0}", times[times.Count/2]);
// }
// private IBlueRow myGetRow (int id)
// {
// return new BlueRow (mySchema,
// Convert.ToByte (2), Convert.ToByte (4), DateTime.Now, id, 10.324555);
// }
// }
// }
@@ -0,0 +1,46 @@
// using System;
// using NUnit.Framework;
// using BCG.Utilities.General;
// namespace BCG
// {
// [TestFixture]
// public class Performance
// {
// public Performance()
// {
// }
// [Test]
// public void testContructionTime ()
// {
// double totalTime;
// string deserName = "BCG.ClientUniversalDeserializer";
// HighPerformanceTimer timer = new HighPerformanceTimer ();
// IClientDeserializer cd;
// for (int i = 0; i < 100; i++)
// {
// timer.start ();
// cd = Activator.CreateInstance (Type.GetType (deserName)) as IClientDeserializer;
// timer.stop ();
// //Console.Write (timer.lastIntervalTime.ToString() + " ");
// }
// Console.WriteLine ("average time = " + timer.totalMilliseconds / 100);
// totalTime = timer.totalMilliseconds;
// timer = new HighPerformanceTimer ();
// for (int i = 0; i < 100; i++)
// {
// timer.start ();
// cd = new ClientUniversalDeserializer ();
// timer.stop ();
// //Console.Write (timer.lastIntervalTime.ToString () + " ");
// }
// Console.WriteLine ("average time = " + timer.totalMilliseconds / 100);
// Console.WriteLine ("Ratio = " + totalTime/timer.totalMilliseconds);
// }
// }
// }
@@ -0,0 +1,275 @@
// using System;
// using System.IO;
// using NUnit.Framework;
// namespace BCG
// {
// [TestFixture]
// public class TableAccumulatorTest
// {
// private IBlueTableSchema mySchema = new BlueTableSchema ("test",
// new ColumnInfo ("id", 0, typeof(int)),
// new ColumnInfo ("x", 1, typeof(int)));
// private SubscriberCore mySubCore = new SubscriberCore ();
// private IBlueRowCollection myNr, myAr, myDr;
// public TableAccumulatorTest() {}
// [SetUp]
// public void tearDown ()
// {
// myNr = myAr = myDr = null;
// }
// #region create
// [Test]
// public void testCreate ()
// {
// TableAccumulator ta = new TableAccumulator ("test", mySubCore);
// IBlueDataSerializer ser = new BlueDataSerializationController ();
// byte[] data;
// BlueTable bt = new BlueTable (mySchema);
// for (int i = 0; i < 100; i++)
// bt.addRow (new BlueRow (mySchema, i, 2*i));
// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
// ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
// ta.update (new Stream[] {new MemoryStream (data)});
// //testing the data
// IBlueRowCollection dr, ar;
// //new rows
// BlueDataUtility.computeDifferences (bt, myNr, out dr, out ar);
// Assert.AreEqual (0, dr.Count);
// Assert.AreEqual (0, ar.Count);
// //added rows
// BlueDataUtility.computeDifferences (bt, myAr, out dr, out ar);
// Assert.AreEqual (0, dr.Count);
// Assert.AreEqual (0, ar.Count);
// //deleted rows;
// Assert.AreEqual (0, myDr.Count);
// ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
// }
// #endregion
// #region single add, delete and change
// [Test]
// public void testAddRemoveChange ()
// {
// TableAccumulator ta = new TableAccumulator ("test", mySubCore);
// IBlueDataSerializer ser = new BlueDataSerializationController ();
// byte[] data;
// BlueTable bt = new BlueTable (mySchema);
// for (int i = 0; i < 100; i++)
// bt.addRow (new BlueRow (mySchema, i, 2*i));
// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
// ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
// ta.update (new Stream[] {new MemoryStream (data)});
// //add
// //add a row with id 100
// bt.clear ();
// bt.addRow (new BlueRow (mySchema, 100, 200));
// ser.serialize (StreamOpCode.add, new IBlueRowCollection[] {bt}, out data);
// ta.update (new Stream[] {new MemoryStream (data)});
// //testing the add data
// Assert.AreEqual (101, myNr.Count);
// for (int i = 0; i < 101; i++)
// myNr.containsID (i);
// Assert.AreEqual (0, myDr.Count);
// Assert.AreEqual (1, myAr.Count);
// Assert.That (myAr.containsID (100));
// //remove
// //remove the row with id 0
// bt.clear ();
// bt.addRow (new BlueRow (mySchema, 0, 0));
// ser.serialize (StreamOpCode.remove, new IBlueRowCollection[] {bt}, out data);
// ta.update (new Stream[] {new MemoryStream (data)});
// //testing remove data
// Assert.AreEqual (100, myNr.Count);
// for (int i = 0; i < 101; i++)
// {
// Assert.That ((i == 0 && !myNr.containsID (i)) || (i != 0 && myNr.containsID (i)));
// }
// Assert.AreEqual (0, myAr.Count);
// Assert.AreEqual (1, myDr.Count);
// Assert.That (myDr.containsID (0));
// //change
// bt.clear ();
// bt.addRow (new BlueRow (mySchema, 100, -100));
// ser.serialize (StreamOpCode.change, new IBlueRowCollection[] {bt}, out data);
// ta.update (new Stream[] {new MemoryStream (data)});
// //testing changed data
// Assert.AreEqual (100, myNr.Count);
// Assert.AreEqual (1, myAr.Count);
// Assert.AreEqual (100, BlueDataUtility.extractRow (myAr, 0)["id"]);
// Assert.AreEqual (-100, BlueDataUtility.extractRow (myAr, 0)["x"]);
// Assert.AreEqual (1, myDr.Count);
// Assert.AreEqual (100, BlueDataUtility.extractRow (myDr, 0)["id"]);
// Assert.AreEqual (200, BlueDataUtility.extractRow (myDr, 0)["x"]);
// ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
// }
// #endregion
// #region multiple change
// public void testMultipleChange ()
// {
// TableAccumulator ta = new TableAccumulator ("test", mySubCore);
// IBlueDataSerializer ser = new BlueDataSerializationController ();
// byte[] data;
// BlueTable bt = new BlueTable (mySchema);
// for (int i = 0; i < 100; i++)
// bt.addRow (new BlueRow (mySchema, i, 2*i));
// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
// ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
// ta.update (new Stream[] {new MemoryStream (data)});
// //partial rows changed
// bt.clear ();
// for (int i = 0; i < 50; i++)
// bt.addRow (new BlueRow (mySchema, i, -2*i));
// ser.serialize (StreamOpCode.multipleChange, new IBlueRowCollection[] {bt}, out data);
// ta.update (new Stream[] {new MemoryStream (data)});
// //checking received data
// Assert.AreEqual (100, myNr.Count);
// foreach (IBlueRow r in myNr)
// {
// if ((int)r["id"] < 50)
// Assert.AreEqual ((int)r["id"] * -2, r["x"]);
// else
// Assert.AreEqual ((int)r["id"] * 2, r["x"]);
// }
// Assert.AreEqual (50, myAr.Count);
// foreach (IBlueRow r in myAr)
// Assert.AreEqual ((int)r["id"] * -2, r["x"]);
// Assert.AreEqual (50, myDr.Count);
// foreach (IBlueRow r in myDr)
// Assert.AreEqual ((int)r["id"] * 2, r["x"]);
// //full rows changed
// bt.clear ();
// for (int i = 0; i < 100; i++)
// bt.addRow (new BlueRow (mySchema, i, -2*i));
// ser.serialize (StreamOpCode.multipleChange, new IBlueRowCollection[] {bt}, out data);
// ta.update (new Stream[] {new MemoryStream (data)});
// //checking received data
// Assert.AreEqual (100, myNr.Count);
// foreach (IBlueRow r in myNr)
// {
// Assert.AreEqual ((int)r["id"] * -2, r["x"]);
// }
// Assert.AreEqual (100, myAr.Count);
// foreach (IBlueRow r in myAr)
// Assert.AreEqual ((int)r["id"] * -2, r["x"]);
// Assert.AreEqual (100, myDr.Count);
// foreach (IBlueRow r in myDr)
// {
// if ((int)r["id"] < 50)
// Assert.AreEqual ((int)r["id"] * -2, r["x"]);
// else
// Assert.AreEqual ((int)r["id"] * 2, r["x"]);
// }
// ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
// }
// #endregion
// #region multiple deletes/adds
// public void testMultipleDeletesAdds ()
// {
// TableAccumulator ta = new TableAccumulator ("test", mySubCore);
// IBlueDataSerializer ser = new BlueDataSerializationController ();
// byte[] data;
// BlueTable bt = new BlueTable (mySchema);
// for (int i = 0; i < 100; i++)
// bt.addRow (new BlueRow (mySchema, i, 2*i));
// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
// ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
// ta.update (new Stream[] {new MemoryStream (data)});
// BlueTable at = new BlueTable (mySchema),
// dt = new BlueTable (mySchema);
// for (int i = 0; i < 50; i++)
// dt.addRow (new BlueRow (mySchema, i, 2*i));
// for (int i = 100; i < 150; i++)
// at.addRow (new BlueRow (mySchema, i, 2*i));
// ser.serialize (StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] {dt, at}, out data);
// ta.update (new Stream[] {new MemoryStream (data)});
// //checking state
// Assert.AreEqual (100, myNr.Count);
// for (int i = 50; i < 150; i++)
// Assert.That (myNr.containsID (i));
// Assert.AreEqual (50, myDr.Count);
// for (int i = 0; i < 50; i++)
// Assert.That (myDr.containsID (i));
// Assert.AreEqual (50, myAr.Count);
// for (int i = 100; i < 150; i++)
// Assert.That (myAr.containsID (i));
// ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
// }
// #endregion
// #region multiple ops in one stream
// [Test]
// public void testMultipleOps ()
// {
// TableAccumulator ta = new TableAccumulator ("test", mySubCore);
// ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
// IBlueDataSerializer ser = new BlueDataSerializationController ();
// byte[] data;
// MemoryStream ms = new MemoryStream ();
// BlueTable bt = new BlueTable (mySchema);
// for (int i = 0; i < 100; i++)
// bt.addRow (new BlueRow (mySchema, i, 2*i));
// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
// ta.update (new Stream[] {new MemoryStream (data)});
// //ms.Write (data, 0, data.Length);
// //changing the first 5 rows
// bt.clear ();
// for (int i = 0; i < 5; i++)
// bt.addRow (new BlueRow (mySchema, i, 1000));
// ser.serialize (StreamOpCode.multipleChange, new IBlueRowCollection[] {bt}, out data);
// ms.Write (data, 0, data.Length);
// //delete id 5
// bt.clear ();
// bt.addRow (new BlueRow (mySchema, 5, 10));
// ser.serialize (StreamOpCode.remove, new IBlueRowCollection[] {bt}, out data);
// ms.Write (data, 0, data.Length);
// //add id 100
// bt.clear ();
// bt.addRow (new BlueRow (mySchema, 100, 200));
// ser.serialize (StreamOpCode.add, new IBlueRowCollection[] {bt}, out data);
// ms.Write (data, 0, data.Length);
// //delete 4 and add 5
// bt.clear ();
// BlueTable dels = new BlueTable (bt.schema);
// for (int i = 90; i < 94; i++)
// dels.addRow (new BlueRow (mySchema, i, i*2));
// for (int i = 101; i < 106; i++)
// bt.addRow (new BlueRow (mySchema, i, 2*i));
// ser.serialize (StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] {dels, bt}, out data);
// ms.Write (data, 0, data.Length);
// ms.Position = 0;
// ta.update (new Stream[] {ms});
// Assert.AreEqual (101, myNr.Count);
// Assert.AreEqual (10, myDr.Count);
// Assert.AreEqual (11, myAr.Count);
// ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
// }
// #endregion
// private void myHandleUpdate (IBlueRowCollection nr, IBlueRowCollection ar, IBlueRowCollection dr)
// {
// myNr = nr;
// myAr = ar;
// myDr = dr;
// }
// }
// }
@@ -0,0 +1,58 @@
// using System;
// using System.IO;
// using NUnit.Framework;
// namespace BCG
// {
// [TestFixture]
// public class UniversalRowSerializerTest
// {
// private IBlueRowSerializer myRowSerializer;
// private IClientRowDeserializer myRowDeserializer;
// private Random myRnd = new Random (DateTime.Now.Millisecond);
// private IBlueTableSchema mySchema = new BlueTableSchema ("Test",
// new ColumnInfo ("id", 0, typeof(int)),
// new ColumnInfo ("int1", 1, typeof(int)),
// new ColumnInfo ("int2", 2, typeof(int)),
// new ColumnInfo ("double1", 3, typeof(double)),
// new ColumnInfo ("double2", 4, typeof(double)),
// new ColumnInfo ("byte1", 5, typeof(byte)),
// new ColumnInfo ("string1", 6, typeof(string)),
// new ColumnInfo ("string2", 7, typeof(string)));
// public UniversalRowSerializerTest () {}
// [Test]
// public void testSerialiazationCycle ()
// {
// myRowSerializer = new UniversalRowSerializer (mySchema);
// myRowDeserializer = new ClientUniversalRowDeserializer (mySchema);
// for (int i = 0; i < 10000; i++)
// {
// BinaryWriter bw = new BinaryWriter (new MemoryStream ());
// IBlueRow ir = myGetRandomRow ();
// myRowSerializer.serializeRow (ir, StreamOpCode.add, bw);
// IBlueRow or;
// (bw.BaseStream as MemoryStream).Position = 0;
// BinaryReader br = new BinaryReader (bw.BaseStream);
// myRowDeserializer.deserializeRow (null, br, StreamOpCode.add, out or);
// Assert.That (BlueDataUtility.computeRowEquality (ir, or));
// }
// }
// private IBlueRow myGetRandomRow ()
// {
// return new BlueRow (mySchema,
// myRnd.Next (1000),
// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : myRnd.Next (1000) as object,
// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object: myRnd.Next (1000) as object,
// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : myRnd.NextDouble () as object,
// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : myRnd.NextDouble () as object,
// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : Convert.ToByte (76) as object,
// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : "abcdefgh" as object,
// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : "1234567890" as object
// );
// }
// }
// }
@@ -0,0 +1,13 @@
PubTableBroker allows publishing of streams through createTable, addRow, deleteRow, changeRow, makeBatchChanges ->
PubTablePublisher:IPubTablePublisher per tag (addRow, deleteRow, changeRow, createTable, batchRemoveAdd)
PubTablePublisher has BlueDataSerializationController:IBlueDataSerializer to serialize the rows to byte[]
BlueDataSerializationController uses a IBlueDataSerializer, which is specialized for the tag (using UniversalSerializer for now)
IBlueDataSerializer has just one function
void serialize (byte op, IBlueRowCollection[] rows, out byte[] serializedData);
the second arg is IBlueRowCollection[] because in case of op = StreamOpCode.multipleRemoveAdd, it has multiple IBlueRowCollection for rem and add
(BlueDataSerializer.serialize's comment has the serialization layout)
For each serializer we have 2 matching deserializers, IPubsubDeserializer (for PubSubservr) and IClientDeserializer (for clients subscribing to streams)
For UniversalSerializer, we have PubsubUniversalDeserializer and ClientUniversalDeserializer
@@ -3,6 +3,9 @@
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\..\client\spark_api\spark_api.csproj" /> <ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" /> <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
<ProjectReference Include="..\pubsub\pubsub.csproj" />
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
<ProjectReference Include="..\DataStream\DataStream.csproj" />
</ItemGroup> </ItemGroup>
<PropertyGroup> <PropertyGroup>
@@ -1,16 +1,35 @@
using bts.utils; using System;
using mbts.config; using BCG;
using System; bts.utils.BTSNativeMethods.provide_command_line(new string[] { }, 0);
using bts; mbts.config.Configuration.initialize();
using mbts;
BTSNativeMethods.provide_command_line(args, args.Length); bts.Log.Gen.info($"Starting DSPublisher");
// Must initialize config before using log system Console.WriteLine("Starting DSPublisher...");
Console.WriteLine("Initializing config..."); // SimpleTCPClient.start("127.0.0.1", 12345);
Configuration.initialize(); // Task.Delay(1000000000).Wait();
for (int i = 0; i < 10; i++)
{ //TestPubliser.publish();
bts.Log.Gen.info($"Starting DataStreamPublisher...log: {i}"); //TestPubliser.subscribe();
// TestPublisher ts = new TestPublisher(false);
// ts.keep_publishing(TimeSpan.FromSeconds(10));
_subscribe_to_stream();
void _subscribe_to_stream() {
BCG.ViewStream vs = new TableStream("literal");
vs.onChange += on_change;
} }
while (true) {
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
ViewStreamDatabase.instance.doWork();
}
void on_change(object obj) {
DataPacket dp = obj as DataPacket;
dp.newRows.print_table((col) => { return 10; });
}
@@ -1,5 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<ItemGroup>
<ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
<ProjectReference Include="..\pubsub\pubsub.csproj" />
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
<ProjectReference Include="..\DataStream\DataStream.csproj" />
<!-- <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" /> -->
</ItemGroup>
<PropertyGroup> <PropertyGroup>
<OutputType>Exe</OutputType> <OutputType>Exe</OutputType>
<TargetFramework>net7.0-windows</TargetFramework> <TargetFramework>net7.0-windows</TargetFramework>
@@ -9,9 +9,11 @@ using mbts;
using mbts.config; using mbts.config;
using bts.utils; using bts.utils;
static class Program { static class Program
{
[STAThread] [STAThread]
static void Main(string[] args) { static void Main(string[] args)
{
BTSNativeMethods.provide_command_line(args, args.Length); BTSNativeMethods.provide_command_line(args, args.Length);
Configuration.initialize(); Configuration.initialize();
bts.Log.Gen.info($"Starting DSQA"); bts.Log.Gen.info($"Starting DSQA");
@@ -21,71 +23,10 @@ static class Program {
// TestPubliser.subscribe(); // TestPubliser.subscribe();
Console.WriteLine("Launching Main Window"); Console.WriteLine("Launching Main Window");
MainForm mainForm = new MainForm(); MainForm mainForm = new MainForm();
if (args.Length > 0) { if (args.Length > 0)
{
mainForm.loadFile(args[0]); mainForm.loadFile(args[0]);
} }
Application.Run(mainForm); Application.Run(mainForm);
} }
} }
class TestPubItem : IFormatter, IPubItem
{
public DateTime send_time = DateTime.Now;
public object Deserialize(Stream str)
{
BinaryReader br = new BinaryReader(str);
var d = br.ReadDouble();
TestPubItem rv = new TestPubItem();
rv.send_time = DateTime.FromOADate(d);
return rv;
}
public void Serialize(Stream str, Object obj)
{
TestPubItem tpi = obj as TestPubItem;
BinaryWriter bw = new BinaryWriter(str);
bw.Write(tpi.send_time.ToOADate());
}
public System.Runtime.Serialization.SerializationBinder? Binder { get; set; }
public System.Runtime.Serialization.StreamingContext Context { get; set; }
public System.Runtime.Serialization.ISurrogateSelector? SurrogateSelector { get; set; }
public string getTag()
{
return "Test_Tag";
}
}
class TestPubliser
{
public static void publish()
{
int n_pubs = 1000;
Publisher<TestPubItem> publisher = new Publisher<TestPubItem>();
for (int i = 0; i < n_pubs; i++)
{
TestPubItem pi = new TestPubItem();
pi.send_time = DateTime.Now;
bts.Log.Gen.info($"DSQA publishing tag: {pi.getTag()}");
publisher.publish(pi);
System.Threading.Thread.Sleep(100);
MasterWorker.getInstance(false).doWork();
}
}
public static void subscribe()
{
ManagedSubscriber ms = new ManagedSubscriber();
ms.subscribe("Test_Tag", new Handler(on_data));
while (true)
{
ms.doWork();
}
}
public static void on_data(IPubItem pi)
{
Console.WriteLine("Received data");
}
}
+5 -13
View File
@@ -21,19 +21,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_core", "..\client\spa
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_api", "..\client\spark_api\spark_api.csproj", "{52DD33DB-00F5-44BF-AE0F-5FC58FB2277E}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_api", "..\client\spark_api\spark_api.csproj", "{52DD33DB-00F5-44BF-AE0F-5FC58FB2277E}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BlueRegistry", "BlueRegistry\BlueRegistry.csproj", "{0FD1BE7A-AEC8-4575-BA58-20E0F45E5214}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamTests", "DataStreamTests\DataStreamTests.csproj", "{66AB10D3-155F-4663-B1BC-D0818E17B87B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "BaseUtilities\BaseUtilities.csproj", "{A9EDEEBD-A03A-4281-8FAF-2419D3E7009C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GuiUtilities", "GuiUtilities\GuiUtilities.csproj", "{CADBA1B3-0837-44DD-85BB-70351AD3A028}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "ArithmeticExpressions\ArithmeticExpressions.csproj", "{EFE6AD17-2564-43A1-B0E5-A2A8DA173002}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "DataStreamExpressions\DataStreamExpressions.csproj", "{D46A152D-AD29-4BEF-980B-5896694E312F}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "ColumnMapExpressions\ColumnMapExpressions.csproj", "{A844AB94-195B-4A7F-89CD-9A335985328E}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "ExpressionCore\ExpressionCore.csproj", "{B94AB04B-76D0-4D37-8F2E-7A57A70D289F}"
EndProject EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -80,6 +68,10 @@ Global
{52DD33DB-00F5-44BF-AE0F-5FC58FB2277E}.Debug|Any CPU.Build.0 = Debug|x64 {52DD33DB-00F5-44BF-AE0F-5FC58FB2277E}.Debug|Any CPU.Build.0 = Debug|x64
{52DD33DB-00F5-44BF-AE0F-5FC58FB2277E}.Release|Any CPU.ActiveCfg = Release|x64 {52DD33DB-00F5-44BF-AE0F-5FC58FB2277E}.Release|Any CPU.ActiveCfg = Release|x64
{52DD33DB-00F5-44BF-AE0F-5FC58FB2277E}.Release|Any CPU.Build.0 = Release|x64 {52DD33DB-00F5-44BF-AE0F-5FC58FB2277E}.Release|Any CPU.Build.0 = Release|x64
{66AB10D3-155F-4663-B1BC-D0818E17B87B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{66AB10D3-155F-4663-B1BC-D0818E17B87B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{66AB10D3-155F-4663-B1BC-D0818E17B87B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{66AB10D3-155F-4663-B1BC-D0818E17B87B}.Release|Any CPU.Build.0 = Release|Any CPU
{0FD1BE7A-AEC8-4575-BA58-20E0F45E5214}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {0FD1BE7A-AEC8-4575-BA58-20E0F45E5214}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0FD1BE7A-AEC8-4575-BA58-20E0F45E5214}.Debug|Any CPU.Build.0 = Debug|Any CPU {0FD1BE7A-AEC8-4575-BA58-20E0F45E5214}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0FD1BE7A-AEC8-4575-BA58-20E0F45E5214}.Release|Any CPU.ActiveCfg = Release|Any CPU {0FD1BE7A-AEC8-4575-BA58-20E0F45E5214}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -0,0 +1,28 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
<PackageReference Include="NUnit" Version="3.13.3" />
<PackageReference Include="NUnit3TestAdapter" Version="4.3.0" />
<PackageReference Include="NUnit.Analyzers" Version="3.5.0" />
<PackageReference Include="coverlet.collector" Version="3.1.2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
<ProjectReference Include="..\pubsub\pubsub.csproj" />
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
<ProjectReference Include="..\DataStream\DataStream.csproj" />
</ItemGroup>
</Project>
@@ -0,0 +1,195 @@
using System;
using System.IO;
using NUnit.Framework;
namespace BCG {
/// <summary>
/// Tests the deserialization happening within the pubsub.
/// </summary>
public class UniversalSerializationTest {
protected IBlueTableSchema mySchema;
protected BlueTable myTable;
protected IBlueDataSerializer mySer = new BlueDataSerializationController();
protected IPubsubDeserializer myPubsubDeser = new PubsubDeserializationController();
protected IClientDeserializer myClientDeser = new ClientDeserializationController();
public UniversalSerializationTest() {
mySchema = new BlueTableSchema(myGetTableName(),
new ColumnInfo("id", 0, typeof(int)),
new ColumnInfo("i1", 1, typeof(int)),
new ColumnInfo("d1", 2, typeof(double)),
new ColumnInfo("dt", 3, typeof(DateTime)),
new ColumnInfo("x", 4, typeof(byte))
);
}
protected virtual string myGetTableName() {
return "testUniversal";
}
/// <summary>
/// Tests the entire cycle. This is done by 'publishing' data to the serializationController (ie a series of
/// adds, deletions, changes etc) and getting the bytes and feeding it into a pubsubDeseriazlier and a clientDeserializer.
/// At every step the data is tested for consistency. At the end the getState is used to get all the data, and its passed
/// through the clientDeserializer to get all the rows.
///
/// </summary>
[Test]
public void testEntireProcess() {
myTable = new BlueTable(mySchema);
byte[] data;
int id;
//create
for (id = 0; id < 100; id++)
myTable.addRow(myGetRow(id));
byte[] ops;
Array rows;
IBlueRowCollection del, add;
IBlueRowCollection[] receivedData;
IBlueRow r;
//create
mySer.serialize(StreamOpCode.create, new IBlueRowCollection[] { myTable }, out data);
myWrite(StreamOpCode.create, data);
myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
Assert.AreEqual(1, ops.Length);
Assert.AreEqual(StreamOpCode.create, ops[0]);
Assert.AreEqual(1, rows.Length);
receivedData = rows.GetValue(0) as IBlueRowCollection[];
Assert.AreEqual(1, receivedData.Length);
BlueDataUtility.computeDifferences(receivedData[0], myTable, out del, out add);
Assert.AreEqual(0, del.Count);
Assert.AreEqual(0, add.Count);
//single add
r = myGetRow(id);
BlueTable temp = new BlueTable(mySchema);
myTable.addRow(r);
temp.addRow(r);
mySer.serialize(StreamOpCode.add, new IBlueRowCollection[] { temp }, out data);
myWrite(StreamOpCode.add, data);
myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
Assert.AreEqual(1, ops.Length);
Assert.AreEqual(StreamOpCode.add, ops[0]);
Assert.AreEqual(1, rows.Length);
receivedData = rows.GetValue(0) as IBlueRowCollection[];
Assert.AreEqual(1, receivedData.Length);
BlueDataUtility.computeDifferences(receivedData[0], temp, out del, out add);
Assert.AreEqual(0, del.Count);
Assert.AreEqual(0, add.Count);
//single delete
r = myTable[0]; //delete row with id 0
temp = new BlueTable(mySchema);
myTable.deleteRow(0);
temp.addRow(r);
mySer.serialize(StreamOpCode.remove, new IBlueRowCollection[] { temp }, out data);
myWrite(StreamOpCode.remove, data);
myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
Assert.AreEqual(1, ops.Length);
Assert.AreEqual(StreamOpCode.remove, ops[0]);
Assert.AreEqual(1, rows.Length);
receivedData = rows.GetValue(0) as IBlueRowCollection[];
Assert.AreEqual(1, receivedData.Length);
BlueDataUtility.computeDifferences(receivedData[0], temp, out del, out add);
Assert.AreEqual(0, del.Count);
Assert.AreEqual(0, add.Count);
//batch deletes and adds
//0 was removed earlier, so lets delete from 1 to 24 first, and then add 101 to 124
BlueTable delRows = new BlueTable(mySchema), addRows = new BlueTable(mySchema);
for (int i = 1; i < 25; i++) {
delRows.addRow(myTable[i]);
myTable.deleteRow(i);
}
for (id = 101; id < 125; id++) {
r = myGetRow(id);
addRows.addRow(r);
myTable.addRow(r);
}
for (int i = 25; i < 125; i++)
Assert.AreEqual(true, myTable.containsID(i));
Assert.AreEqual(100, myTable.Count);
mySer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { delRows, addRows }, out data);
myWrite(StreamOpCode.multipleRemoveAdd, data);
myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
Assert.AreEqual(1, ops.Length);
Assert.AreEqual(StreamOpCode.multipleRemoveAdd, ops[0]);
Assert.AreEqual(1, rows.Length);
receivedData = rows.GetValue(0) as IBlueRowCollection[];
Assert.AreEqual(2, receivedData.Length);
BlueDataUtility.computeDifferences(receivedData[0], delRows, out del, out add);
Assert.AreEqual(0, del.Count);
Assert.AreEqual(0, add.Count);
BlueDataUtility.computeDifferences(receivedData[1], addRows, out del, out add);
Assert.AreEqual(0, del.Count);
Assert.AreEqual(0, add.Count);
//pure changes only
//change the first 25, 25-49
BlueTable changedRows = new BlueTable(mySchema);
for (int i = 25; i < 50; i++) {
MutableBlueRow mr = new MutableBlueRow(myTable[i]);
mr.setValue(4, Byte.MaxValue);
myTable.changeRow(mr);
changedRows.addRow(mr);
}
mySer.serialize(StreamOpCode.multipleChange, new IBlueRowCollection[] { changedRows }, out data);
myWrite(StreamOpCode.multipleChange, data);
myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
Assert.AreEqual(1, ops.Length);
Assert.AreEqual(StreamOpCode.multipleChange, ops[0]);
Assert.AreEqual(1, rows.Length);
receivedData = rows.GetValue(0) as IBlueRowCollection[];
Assert.AreEqual(1, receivedData.Length);
BlueDataUtility.computeDifferences(receivedData[0], changedRows, out del, out add);
Assert.AreEqual(0, del.Count);
Assert.AreEqual(0, add.Count);
//final test
data = myPubsubDeser.getState();
new ClientDeserializationController().deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
Assert.AreEqual(1, ops.Length);
Assert.AreEqual(StreamOpCode.create, ops[0]);
Assert.AreEqual(1, rows.Length);
receivedData = rows.GetValue(0) as IBlueRowCollection[];
Assert.AreEqual(1, receivedData.Length);
BlueDataUtility.computeDifferences(receivedData[0], myTable, out del, out add);
Assert.AreEqual(0, del.Count);
Assert.AreEqual(0, add.Count);
}
private IBlueRow myGetRow(int id) {
return new BlueRow(mySchema, // the day pp fixed this test.
id, id + 10, id * 3.0, new DateTime(2008, 6, 4, 0, 0, 0, DateTimeKind.Unspecified), Convert.ToByte(id));
}
private void myWrite(byte op, byte[] data) {
string s = "";
if (op == StreamOpCode.add)
s = "add one row";
else if (op == StreamOpCode.change)
s = "change one row";
else if (op == StreamOpCode.create)
s = "create";
else if (op == StreamOpCode.multipleChange)
s = "multiple change";
else if (op == StreamOpCode.multipleRemoveAdd)
s = "multiple add remove";
else if (op == StreamOpCode.remove)
s = "remove one row";
Console.WriteLine("yes, here , op = {0}, size = {1}", s, data.Length);
}
}
}
@@ -0,0 +1 @@
global using NUnit.Framework;
@@ -4,15 +4,13 @@ using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
using BCG.Utilities; using BCG.Utilities;
namespace BCG namespace BCG {
{
/// <summary> /// <summary>
/// Holds the state for a single tag. If the tagType has an /// Holds the state for a single tag. If the tagType has an
/// IPubSubServerTagHandler, it is updated and the state stored /// IPubSubServerTagHandler, it is updated and the state stored
/// there. Otherwise, this class saves the last published data. /// there. Otherwise, this class saves the last published data.
/// </summary> /// </summary>
internal class SingleTagHandler internal class SingleTagHandler {
{
private IPubSubServerTagHandler myHandler = null; private IPubSubServerTagHandler myHandler = null;
private string myTagType, myTag; private string myTagType, myTag;
/// <summary> /// <summary>
@@ -20,9 +18,7 @@ namespace BCG
/// </summary> /// </summary>
private byte[] myData = null; private byte[] myData = null;
public SingleTagHandler(string tag, string tagType) public SingleTagHandler(string tag, string tagType) {
{
myTagType = tagType; myTagType = tagType;
myTag = tag; myTag = tag;
myHandler = TagType2HandlerMap.getHandler(tagType, false); myHandler = TagType2HandlerMap.getHandler(tagType, false);
@@ -30,23 +26,18 @@ namespace BCG
public string tagType { get { return myTagType; } } public string tagType { get { return myTagType; } }
public void publish(Stream data) public void publish(Stream data) {
{
long pos = data.Position; long pos = data.Position;
if (myHandler != null) if (myHandler != null) {
{
myHandler.update(myTag, new Stream[] { data }); myHandler.update(myTag, new Stream[] { data });
} } else {
else
{
myData = new byte[data.Length]; myData = new byte[data.Length];
data.Read(myData, 0, myData.Length); data.Read(myData, 0, myData.Length);
} }
data.Position = pos; data.Position = pos;
} }
public byte[] getContent() public byte[] getContent() {
{
return myHandler == null ? myData : myHandler.getContent(myTag); return myHandler == null ? myData : myHandler.getContent(myTag);
} }
} }
@@ -54,46 +45,37 @@ namespace BCG
/// <summary> /// <summary>
/// Keeps the state of all the tags published. /// Keeps the state of all the tags published.
/// </summary> /// </summary>
public class AllPublishedTagState public class AllPublishedTagState {
{
private Dictionary<string, SingleTagHandler> myTagHandler = new Dictionary<string, SingleTagHandler>(); private Dictionary<string, SingleTagHandler> myTagHandler = new Dictionary<string, SingleTagHandler>();
public AllPublishedTagState() { } public AllPublishedTagState() { }
public void publish(string tag, string tagType, Stream data) public void publish(string tag, string tagType, Stream data) {
{
if (!myTagHandler.ContainsKey(tag)) if (!myTagHandler.ContainsKey(tag))
myTagHandler.Add(tag, new SingleTagHandler(tag, tagType)); myTagHandler.Add(tag, new SingleTagHandler(tag, tagType));
myTagHandler[tag].publish(data); myTagHandler[tag].publish(data);
} }
public void publish(string tag, string tagType, byte[] data) public void publish(string tag, string tagType, byte[] data) {
{
publish(tag, tagType, new MemoryStream(data)); publish(tag, tagType, new MemoryStream(data));
} }
public string[] publishedTags public string[] publishedTags { get { return new ArrayList(myTagHandler.Keys).ToArray(typeof(string)) as string[]; } }
{ get { return new ArrayList(myTagHandler.Keys).ToArray(typeof(string)) as string[]; } }
public byte[] getContent(string tag, out string tagType) public byte[] getContent(string tag, out string tagType) {
{
tagType = myTagHandler[tag].tagType; tagType = myTagHandler[tag].tagType;
return myTagHandler[tag].getContent(); return myTagHandler[tag].getContent();
} }
public bool contains(string tag) public bool contains(string tag) {
{ return myTagHandler.ContainsKey(tag);
return myTagHandler.ContainsKey(tag);
} }
public void clear(string tag) public void clear(string tag) {
{ if (myTagHandler.ContainsKey(tag)) {
if (myTagHandler.ContainsKey(tag))
{
myTagHandler.Remove(tag); myTagHandler.Remove(tag);
} } else { }
else{} // BlueEventLog.logEvent("AllPublishTagState told to clear nonexisting tag " + tag, BlueEventLog2Level.Warning);
// BlueEventLog.logEvent("AllPublishTagState told to clear nonexisting tag " + tag, BlueEventLog2Level.Warning);
} }
} }
@@ -1,117 +1,117 @@
using System; // using System;
using System.Collections.Generic; // using System.Collections.Generic;
using System.Text; // using System.Text;
using System.Runtime.InteropServices; // using System.Runtime.InteropServices;
using mbts; // using mbts;
//using BCG.Utilities; // //using BCG.Utilities;
namespace BCG // namespace BCG
{ // {
public class PubSubServerUnmanagedTCPConnection : IPubSubServerConnection // public class PubSubServerUnmanagedTCPConnection : IPubSubServerConnection
{ // {
private BlutoSubAddress myServerAddress; // private BlutoSubAddress myServerAddress;
private SmartConnectionManagedWrapper myWrapper; // private SmartConnectionManagedWrapper myWrapper;
private ConnectionStatusChange myConnCallback; // private ConnectionStatusChange myConnCallback;
private SyncQueue<Pair<int, byte[]>> myAllMsgsQueue = new SyncQueue<Pair<int, byte[]>>(); // private SyncQueue<Pair<int, byte[]>> myAllMsgsQueue = new SyncQueue<Pair<int, byte[]>>();
public PubSubServerUnmanagedTCPConnection(string serverURL, SmartConnectionManagedWrapper wrapper) // public PubSubServerUnmanagedTCPConnection(string serverURL, SmartConnectionManagedWrapper wrapper)
{ // {
myServerAddress = new BlutoSubAddress(serverURL); // myServerAddress = new BlutoSubAddress(serverURL);
myWrapper = wrapper; // myWrapper = wrapper;
myWrapper.initConnection(new BlutoSubAddress(serverURL)); // myWrapper.initConnection(new BlutoSubAddress(serverURL));
myWrapper.resgisterDataCallback(myServerAddress, new SmartConnectionManagedWrapper.MessageDelegate(myHandleServerMessages)); // myWrapper.resgisterDataCallback(myServerAddress, new SmartConnectionManagedWrapper.MessageDelegate(myHandleServerMessages));
} // }
#region IPubSubServerConnection Members // #region IPubSubServerConnection Members
public void sendWithConfirmation(byte[] data, ref string confId) // public void sendWithConfirmation(byte[] data, ref string confId)
{ // {
//BlueEventLog.logAssert(false, "This function should not be used on this class"); // //BlueEventLog.logAssert(false, "This function should not be used on this class");
} // }
public void send(byte[] data) // public void send(byte[] data)
{ // {
myWrapper.send(myServerAddress, data); // myWrapper.send(myServerAddress, data);
} // }
public void send(System.IO.Stream str) // public void send(System.IO.Stream str)
{ // {
send (StreamUtilityMethods.getBytesFromStream(str)); // send (StreamUtilityMethods.getBytesFromStream(str));
} // }
public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements) // public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements)
{ // {
List<Pair<int, byte[]>> msgs = myAllMsgsQueue.dequeueMultipleElements(Convert.ToInt32 (timeout.TotalMilliseconds), maxElements); // List<Pair<int, byte[]>> msgs = myAllMsgsQueue.dequeueMultipleElements(Convert.ToInt32 (timeout.TotalMilliseconds), maxElements);
List<byte[]> rv = new List<byte[]>(); // List<byte[]> rv = new List<byte[]>();
bool connSet = false; // bool connSet = false;
foreach (Pair<int, byte[]> p in msgs) // foreach (Pair<int, byte[]> p in msgs)
{ // {
int msgType = p.first; // int msgType = p.first;
byte[] msg = p.second; // byte[] msg = p.second;
if (msgType == 1) // if (msgType == 1)
{ // {
rv.Add(msg); // rv.Add(msg);
} // }
else // else
{ // {
connSet = true; // connSet = true;
string connMsg = "connection to " + myServerAddress.ToString() + (msgType == 2 ? " broken" : " established"); // string connMsg = "connection to " + myServerAddress.ToString() + (msgType == 2 ? " broken" : " established");
Console.WriteLine(connMsg); // Console.WriteLine(connMsg);
myConnCallback(myServerAddress.ToString(), msgType == 3); // myConnCallback(myServerAddress.ToString(), msgType == 3);
} // }
} // }
return connSet ? new List<byte[]>() : rv; // return connSet ? new List<byte[]>() : rv;
} // }
public string serverURL
{
get { return myServerAddress.ToString(); }
}
public string clientURL // public string serverURL
{ // {
get { return myWrapper.getClientAsSeenByServer (serverURL); } // get { return myServerAddress.ToString(); }
} // }
public void registerCallback(ConnectionStatusChange del) // public string clientURL
{ // {
myConnCallback = del; // get { return myWrapper.getClientAsSeenByServer (serverURL); }
} // }
public void registerDataCallback(DataDelegate del) // public void registerCallback(ConnectionStatusChange del)
{ // {
} // myConnCallback = del;
// }
public void registerSendConfirmationCallback(SendConfirmationDelegate del) // public void registerDataCallback(DataDelegate del)
{ // {
} // }
#endregion // public void registerSendConfirmationCallback(SendConfirmationDelegate del)
// {
// }
#region IDisposable Members // #endregion
public void Dispose() // #region IDisposable Members
{
}
#endregion // public void Dispose()
// {
#region IWorker Members // }
public void doWork()
{
}
#endregion
/// <summary> // #endregion
/// this is the only method that it executed on
/// a different thread // #region IWorker Members
/// </summary> // public void doWork()
/// <param name="data"></param> // {
/// <returns></returns> // }
private void myHandleServerMessages(int msgType, byte[] msg) // #endregion
{
myAllMsgsQueue.enqueue(new Pair<int, byte[]>(msgType, msg)); // /// <summary>
} // /// this is the only method that it executed on
} // /// a different thread
} // /// </summary>
// /// <param name="data"></param>
// /// <returns></returns>
// private void myHandleServerMessages(int msgType, byte[] msg)
// {
// myAllMsgsQueue.enqueue(new Pair<int, byte[]>(msgType, msg));
// }
// }
// }
@@ -4,8 +4,7 @@ using System.Text;
using System.IO; using System.IO;
using BCG.Utilities.General; using BCG.Utilities.General;
namespace BCG namespace BCG {
{
/// <summary> /// <summary>
/// This is the interface that both the PublisherCore and the /// This is the interface that both the PublisherCore and the
/// SubscriberCore use to communicate with the PubsubServer. /// SubscriberCore use to communicate with the PubsubServer.
@@ -20,22 +19,23 @@ namespace BCG
/// multiple threads need to communicate to the PubSub, they /// multiple threads need to communicate to the PubSub, they
/// should create their own Subscribers. /// should create their own Subscribers.
/// </summary> /// </summary>
public interface IPubSubServerConnection : IDisposable, IWorker public interface IPubSubServerConnection : IDisposable, IWorker {
{
/// <summary> /// <summary>
/// The delegate is fired signalling if the data send was /// The delegate is fired signalling if the data send was
/// successful or not /// successful or not
/// </summary> /// </summary>
void sendWithConfirmation(byte[] data, ref string confId); //void sendWithConfirmation(byte[] data, ref string confId);
/// <summary> /// <summary>
/// send is a non waiting call with no guarantee /// send is a non waiting call with no guarantee
/// of data being delivered /// of data being delivered
/// </summary> /// </summary>
void send (byte[] data); // void send(byte[] data);
/// <summary> // /// <summary>
/// similiar to above // /// similiar to above
/// </summary> // /// </summary>
void send(Stream str); void send(Stream str);
void send(PubSubMessage msg);
void send_with_confirmation(PubSubMessage msg, ref string confId);
/// <summary> /// <summary>
/// Returns with 0 or more elements, iff the timeout /// Returns with 0 or more elements, iff the timeout
/// expires or maxElements have been reached (selective polling), /// expires or maxElements have been reached (selective polling),
@@ -51,13 +51,13 @@ namespace BCG
/// <summary> /// <summary>
/// server URL to which the connection is made /// server URL to which the connection is made
/// </summary> /// </summary>
string serverURL { get;} string serverURL { get; }
/// <summary> /// <summary>
/// client URL as seen by the server. if a connection has not /// client URL as seen by the server. if a connection has not
/// been established (as can happen in TCP) an empty string /// been established (as can happen in TCP) an empty string
/// is returned /// is returned
/// </summary> /// </summary>
string clientURL { get;} string clientURL { get; }
/// <summary> /// <summary>
/// registers the callback to receive /// registers the callback to receive
@@ -25,48 +25,64 @@ namespace BCG
ITCPMessageHandler, ITCPMessageHandler,
ITCPConnectionHandler ITCPConnectionHandler
{ {
private ITCPConnection myConnection, myTempNewConnection; private ITCPConnection _connection, _new_connection;
private BlutoSubAddress myServerAddress; private readonly BlutoSubAddress _server_address;
private ThreadSafeQueue<Pair<string, bool>> myConfs = new ThreadSafeQueue<Pair<string, bool>>(); // if a message(with confirm) can not be sent , its id is queued here,
private SyncQueue<byte[]> myDataQueue = new SyncQueue<byte[]>(); // on reconnection, _on_send_onfirm_callback is fired with a negative on these ids, so the user
private DateTime myLastHB = DateTime.Now; //can re-send it. The re-send mechanism is not in implemented in this class, but by the user of this class
private ConnectionStatusChange myConnCallback; private ThreadSafeQueue<string> _negative_unsent_confs = new ThreadSafeQueue<string>();
private DataDelegate myDataCallback; // in onData, data is queued here
private SendConfirmationDelegate myOnSendConfirmCallback; private SyncQueue<byte[]> _in_data_queue = new SyncQueue<byte[]>();
private TimeSpan myConnectionDropThreshold = private DateTime _last_hb = DateTime.Now;
private ConnectionStatusChange _conn_callback;
private DataDelegate _data_callback;
private SendConfirmationDelegate _on_send_onfirm_callback;
private readonly TimeSpan _connection_drop_threshold =
new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval * PubSub3Config.instance.PS3_MissingHeartBeatsForGC); new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval * PubSub3Config.instance.PS3_MissingHeartBeatsForGC);
public PubSubServerManagedTCPConnection(string serverURL) public PubSubServerManagedTCPConnection(string serverURL)
: this(new BlutoSubAddress(serverURL)) { } : this(new BlutoSubAddress(serverURL)) { }
public PubSubServerManagedTCPConnection(BlutoSubAddress serverAddress) public PubSubServerManagedTCPConnection(BlutoSubAddress server_address)
{ {
myServerAddress = serverAddress; _server_address = server_address;
ClientToServerManagedTCPConnector.connectTo(serverAddress, this); _connect_to_ps_server();
}
private void _connect_to_ps_server()
{
bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server: {_server_address}");
ClientToServerManagedTCPConnector.connect_to(_server_address, this, TimeSpan.FromSeconds(1));
} }
#region IPubSubServerConnection Members #region IPubSubServerConnection Members
public void sendWithConfirmation(byte[] data, ref string confId) public void sendWithConfirmation(byte[] data, ref string conf_id)
{ {
if (myConnection != null) bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} myConnection : {_connection != null}");
if (_connection != null)
{ {
myConnection.sendWithConfirmation(data, ref confId, new SendConfirmationDelegate(myOnSendConfirm)); _connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) =>
{
if (_on_send_onfirm_callback != null)
_on_send_onfirm_callback(msg_id, confirmed);
});
} }
else else
{ {
confId = Guid.NewGuid().ToString(); bts.Log.Gen.warn($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} no connection, buffering send");
myConfs.enqueue(new Pair<string, bool>(confId, false)); conf_id = Guid.NewGuid().ToString();
_negative_unsent_confs.enqueue(conf_id);
} }
} }
public void send(byte[] data) public void send(byte[] data)
{ {
Console.WriteLine($"PubSubServerManagedTCPConnection.send server: {myServerAddress} data: {data.Length} myConnection : {myConnection != null}"); bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_address} data: {data.Length} myConnection : {_connection != null}");
if (myConnection != null) if (_connection != null)
{ {
Console.WriteLine("PubSubServerManagedTCPConnection.send server: " + myServerAddress + "data: " + data.Length); bts.Log.Gen.info("PubSubServerManagedTCPConnection.send server: " + _server_address + "data: " + data.Length);
myConnection.send(data); _connection.send(data);
} }
} }
@@ -76,14 +92,25 @@ namespace BCG
send(br.ReadBytes(Convert.ToInt32(str.Length - str.Position))); send(br.ReadBytes(Convert.ToInt32(str.Length - str.Position)));
} }
public void send(PubSubMessage msg)
{
send(msg.to_stream());
}
public void send_with_confirmation(PubSubMessage msg, ref string conf_id)
{
var bytes = msg.to_bytes();
sendWithConfirmation(bytes, ref conf_id);
}
public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements) public IList<byte[]> getUpdates(TimeSpan timeout, int maxElements)
{ {
return myDataQueue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements); return _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements);
} }
public string serverURL public string serverURL
{ {
get { return myServerAddress.ToString(); } get { return _server_address.ToString(); }
} }
public string clientURL public string clientURL
@@ -93,7 +120,7 @@ namespace BCG
string rv = ""; string rv = "";
lock (this) lock (this)
{ {
if (myConnection == null) if (_connection == null)
{ {
rv = ""; rv = "";
} }
@@ -103,7 +130,7 @@ namespace BCG
{ {
string ip; string ip;
int port; int port;
myConnection.getSelfAddress(out ip, out port); _connection.getSelfAddress(out ip, out port);
rv = new BlutoSubAddress(ip, port).ToString(); rv = new BlutoSubAddress(ip, port).ToString();
} }
else else
@@ -118,40 +145,40 @@ namespace BCG
public void registerCallback(ConnectionStatusChange del) public void registerCallback(ConnectionStatusChange del)
{ {
myConnCallback = del; _conn_callback = del;
} }
public void registerDataCallback(DataDelegate del) public void registerDataCallback(DataDelegate del)
{ {
myDataCallback = del; _data_callback = del;
} }
public void registerSendConfirmationCallback(SendConfirmationDelegate del) public void registerSendConfirmationCallback(SendConfirmationDelegate del)
{ {
myOnSendConfirmCallback = del; _on_send_onfirm_callback = del;
} }
#endregion #endregion
#region IDisposable Members #region IDisposable Members
private bool myIsDisposed = false; private bool _is_disposed = false;
public void Dispose() public void Dispose()
{ {
Console.WriteLine("PubSubServerManagedTCPConnection.Dispose"); bts.Log.Gen.info("PubSubServerManagedTCPConnection.Dispose");
//throw new Exception("The method or operation is not implemented."); //throw new Exception("The method or operation is not implemented.");
if (myConnection != null) if (_connection != null)
{ {
(myConnection as AsyncTCPConnection).Dispose(); _connection.Dispose();
} }
myIsDisposed = true; _is_disposed = true;
} }
public bool isDisposed public bool isDisposed
{ {
get get
{ {
return myIsDisposed; return _is_disposed;
} }
} }
@@ -168,17 +195,40 @@ namespace BCG
/// <param name="message"></param> /// <param name="message"></param>
public void onData(RemoteMessage message) public void onData(RemoteMessage message)
{ {
Console.WriteLine($"PubSubServerManagedTCPConnection.onData msg:type: {message.messageType}"); bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData message: {message}");
if (message.messageType == RemoteMessageType.data) // TODO check for disconenction
switch (message.messageType)
{ {
if (myDataCallback != null) case RemoteMessageType.Data:
myDataCallback(message.remoteConnectionInfo.ToString(), message.message); if (_data_callback != null)
_data_callback(message.remoteConnectionInfo.ToString(), message.message);
else
_in_data_queue.enqueue(message.message);
break;
case RemoteMessageType.Heartbeat:
_last_hb = DateTime.Now;
break;
case RemoteMessageType.Connection:
_conn_callback(_server_address.ToString(), true);
break;
case RemoteMessageType.Disconnection:
if (_conn_callback != null)
_conn_callback(_server_address.ToString(), false);
_connection.Dispose();
_connect_to_ps_server();
break;
}
if (message.messageType == RemoteMessageType.Data)
{
if (_data_callback != null)
_data_callback(message.remoteConnectionInfo.ToString(), message.message);
else else
myDataQueue.enqueue(message.message); _in_data_queue.enqueue(message.message);
} }
else else
{ {
myLastHB = DateTime.Now; _last_hb = DateTime.Now;
} }
} }
@@ -197,16 +247,56 @@ namespace BCG
int port; int port;
connection.getPeerAddress(out ip, out port); connection.getPeerAddress(out ip, out port);
BlutoSubAddress add = new BlutoSubAddress(ip, port); BlutoSubAddress add = new BlutoSubAddress(ip, port);
Console.WriteLine($"PubSubServerManagedTCPConnection.onConnection add: {add.ToString()}"); bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onConnection add: {add.ToString()} connection type: {connection.GetType()}");
if (add.ToString() == myServerAddress.ToString()) if (add.ToString() == _server_address.ToString())
{ {
myLastHB = DateTime.Now; _last_hb = DateTime.Now;
myTempNewConnection = connection; _new_connection = connection;
} }
} }
#endregion #endregion
void _dispose_and_reconnect_if_missing_hb()
{
if (!_connection.isDisposed)
{
TimeSpan hb_interval = DateTime.Now - _last_hb;
bool hb_missed = hb_interval > _connection_drop_threshold;
if (hb_missed)
{
if (_conn_callback != null)
_conn_callback(_server_address.ToString(), false);
bts.Log.Gen.info("PubSubServerManagedTCPConnection._dispose_and_reconnect_if_missing_hb disposing connecion: {myServerAddress}");
_connection.Dispose();
_connect_to_ps_server();
}
}
else
{
// already disposed, nothing to do
}
}
void _check_for_new_connection()
{
if (_new_connection != null)
{
_connection = _new_connection;
_new_connection = null; // since we are using its presence as a marker of nw connections
bts.Log.Gen.info("PubSubServerManagedTCPConnection._check_for_new_connection assigning new connection");
_connection.register(this);
if (_conn_callback != null)
_conn_callback(_server_address.ToString(), true);
if (_on_send_onfirm_callback != null)
{
var msg_ids = _negative_unsent_confs.dequeueAll();
foreach (var id in msg_ids)
_on_send_onfirm_callback(id, false);
}
}
}
#region IWorker Members #region IWorker Members
/// <summary> /// <summary>
@@ -216,55 +306,56 @@ namespace BCG
/// </summary> /// </summary>
public void doWork() public void doWork()
{ {
//_dispose_and_reconnect_if_missing_hb();
_check_for_new_connection();
///check for heartbeats and let clients know in ///check for heartbeats and let clients know in
///case of missing hb (connection breaking) ///case of missing hb (connection breaking)
TimeSpan curr_hb_interval = DateTime.Now - myLastHB; // TimeSpan hb_interval = DateTime.Now - myLastHB;
bool hb_missed = curr_hb_interval > myConnectionDropThreshold; // bool hb_missed = hb_interval > myConnectionDropThreshold;
bool conn_disposed = myConnection != null && myConnection.isDisposed; // bool conn_disposed = myConnection != null || myConnection.isDisposed;
if (conn_disposed || hb_missed) // if (conn_disposed || hb_missed)
{ // {
if (hb_missed) // if (hb_missed)
{ // {
// since conn breakage was diagnosed logically, lets physically close the connection // // since conn breakage was diagnosed logically, lets physically close the connection
myConnection.Dispose(); // myConnection.Dispose();
string dispose_msg = $"PubSubServerManagedTCPConnection.doWork server: {myServerAddress} Disposing connection conn_disposed: {conn_disposed} hb_missed: {hb_missed}"; // string dispose_msg = $"PubSubServerManagedTCPConnection.doWork server: {myServerAddress} Disposing connection conn_disposed: {conn_disposed} hb_missed: {hb_missed}";
Console.WriteLine(dispose_msg); // Console.WriteLine(dispose_msg);
} // }
myConnection = null; // this is used as a marker for a dead connection // myConnection = null; // this is used as a marker for a dead connection
if (myConnCallback != null) // if (myConnCallback != null)
myConnCallback(myServerAddress.ToString(), false); // myConnCallback(myServerAddress.ToString(), false);
string reconn_msg = $"PubSubServerManagedTCPConnection.doWork disconnecting client from server: {myServerAddress} conn_disposed: {conn_disposed} hb_missed: {hb_missed} " + // string reconn_msg = $"PubSubServerManagedTCPConnection.doWork disconnecting client from server: {myServerAddress} conn_disposed: {conn_disposed} hb_missed: {hb_missed} " +
$"because of missing hbs last_hb: {myLastHB}, now: {DateTime.Now} curr_hb_interval: {curr_hb_interval.TotalSeconds} threshold: {myConnectionDropThreshold.TotalSeconds} conn_disposed: {conn_disposed}"; // $"because of missing hbs last_hb: {myLastHB}, now: {DateTime.Now} curr_hb_interval: {hb_interval.TotalSeconds} threshold: {myConnectionDropThreshold.TotalSeconds} conn_disposed: {conn_disposed}";
Console.WriteLine(reconn_msg); // ClientToServerManagedTCPConnector.connectTo(myServerAddress, this);
ClientToServerManagedTCPConnector.connectTo(myServerAddress, this); // }
} // if (myTempNewConnection != null)
if (myTempNewConnection != null) // {
{ // if (myConnection != null)
if (myConnection != null) // {
{ // // TODOds
// TODOds // // log error and close it
// log error and close it // // BlueEventLog.logAssert(myConnection == null, "Connection has to be nulled out and disposed "
// BlueEventLog.logAssert(myConnection == null, "Connection has to be nulled out and disposed " // // + " before a new connection is established");
// + " before a new connection is established"); // }
} // Console.WriteLine("PubSubServerManagedTCPConnection hooking new connection");
Console.WriteLine("PubSubServerManagedTCPConnection hooking new connection"); // myConnection = myTempNewConnection;
myConnection = myTempNewConnection; // myTempNewConnection = null;
myTempNewConnection = null; // myConnection.register(this);
myConnection.register(this); // myConfs.dequeueAll();
myConfs.dequeueAll(); // if (myConnCallback != null)
if (myConnCallback != null) // myConnCallback(myServerAddress.ToString(), true);
myConnCallback(myServerAddress.ToString(), true); // }
} // foreach (Pair<string, bool> p in myConfs.dequeueAll())
foreach (Pair<string, bool> p in myConfs.dequeueAll()) // if (myOnSendConfirmCallback != null)
if (myOnSendConfirmCallback != null) // myOnSendConfirmCallback(p.first, p.second);
myOnSendConfirmCallback(p.first, p.second);
} }
#endregion #endregion
private void myOnSendConfirm(string messageId, bool confirmed) // private void myOnSendConfirm(string messageId, bool confirmed) {
{ // _negative_confs.enqueue(new Pair<string, bool>(messageId, confirmed));
myConfs.enqueue(new Pair<string, bool>(messageId, confirmed)); // }
}
} }
} }
@@ -7,144 +7,116 @@ using System.Text;
using BCG; using BCG;
//using BCG.Utilities; using System.Collections.Generic;
namespace BCG namespace BCG {
{ // Thhis "wraps" around the following 4 PubSub meta objects, serializes it on the client and
public class MetaData : ICloneable // deserialize it on the pubsub
{ // PubMetaData
private Hashtable myHash = new Hashtable(); // SubMetaData
// TransactionMetaData
// UnSubMetaData
public class MetaData : ICloneable {
private IDictionary _name_value_map = new Dictionary<string, string>();
int mySize = -1; int mySize = -1;
public MetaData() public MetaData() { }
{}
public MetaData(object o) public MetaData(object o) { copyFrom(o); }
{copyFrom(o);}
public MetaData(Stream s) public MetaData(Stream s) {
{
long initPos = s.Position; long initPos = s.Position;
load(s); load(s);
mySize = Convert.ToInt32(s.Position - initPos); mySize = Convert.ToInt32(s.Position - initPos);
} }
public string this[string name] public string this[string name] {
{ get { return (string)_name_value_map[name]; }
get set {
{return (string) myHash[name];}
set
{
if (value != null) if (value != null)
myHash[name] = value; _name_value_map[name] = value;
} }
} }
public void add(string name, string [] values) private void _add_string_array(string name, string[] values) {
{
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
foreach (string s in values) foreach (string s in values) {
{
if (sb.Length != 0) if (sb.Length != 0)
sb.Append(" "); sb.Append(" ");
sb.Append(s); sb.Append(s);
} }
myHash[name] = sb.ToString(); _name_value_map[name] = sb.ToString();
} }
public int getAsInt(string name) private int getAsInt(string name) { return Int32.Parse(this[name]); }
{return Int32.Parse(this[name]);}
public string[] getAsStringArray (string name) private string[] _get_as_string_array(string name) {
{
if (this[name] != "" && this[name] != null) if (this[name] != "" && this[name] != null)
return this[name].Split(' '); return this[name].Split(' ');
else else
return new string[0]; return new string[0];
} }
public int size() public int size() {
{ if (mySize < 0) {
if (mySize < 0)
{
MemoryStream ms = new MemoryStream(); MemoryStream ms = new MemoryStream();
this.save(ms); this.save(ms);
return mySize = (int)ms.Position; return mySize = (int)ms.Position;
} } else
else
return mySize; return mySize;
} }
public bool isEqual (MetaData md) private bool isEqual(MetaData md) {
{ if (md._name_value_map.Keys.Count != _name_value_map.Keys.Count)
if (md.myHash.Keys.Count != myHash.Keys.Count)
return false; return false;
foreach (string k in myHash.Keys) foreach (string k in _name_value_map.Keys)
if (!myHash[k].Equals(md.myHash[k])) if (!_name_value_map[k].Equals(md._name_value_map[k]))
return false; return false;
return true; return true;
} }
public void load(Stream s) // loads the content of the stream into this instance's map
{ private void load(Stream s) {
myHash = new Hashtable(); _name_value_map = new Hashtable();
string [] state = string[] state =
new BinaryReader(s).ReadString().Split('\n'); new BinaryReader(s).ReadString().Split('\n');
for (int i = 0; i < state.Length-1; i+= 2) for (int i = 0; i < state.Length - 1; i += 2)
myHash[state[i]] = state[i+1]; _name_value_map[state[i]] = state[i + 1];
} }
public void save(Stream s) // writes the content of this instance to stream s
{ public void save(Stream s) {
new BinaryWriter(s).Write(this.ToString()); new BinaryWriter(s).Write(this.ToString());
} }
public override string ToString() public override string ToString() {
{
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
foreach (string key in myHash.Keys) foreach (string key in _name_value_map.Keys)
sb.AppendFormat("{0}\n{1}\n",key,myHash[key]); sb.AppendFormat("{0}\n{1}\n", key, _name_value_map[key]);
return sb.ToString(); return sb.ToString();
} }
public void copyFrom(object o) // iterates through the fields and enters the name, values into _hash
{ public void copyFrom(object o) {
this["op-code"] = o.GetType().Name; this["op-code"] = o.GetType().Name;
foreach (FieldInfo fi in o.GetType().GetFields()) {
foreach (FieldInfo fi in o.GetType().GetFields())
{
if (fi.GetValue(o) == null) if (fi.GetValue(o) == null)
continue; continue;
else if (fi.GetValue(o) is String[]) else if (fi.GetValue(o) is String[])
add(fi.Name, (string[])fi.GetValue(o)); _add_string_array(fi.Name, (string[])fi.GetValue(o));
else else
myHash[fi.Name] = fi.GetValue(o).ToString(); _name_value_map[fi.Name] = fi.GetValue(o).ToString();
} }
} }
// copies the name value map into object o
public void copyTo(object o) public void copyTo(object o) {
{ foreach (FieldInfo fi in o.GetType().GetFields()) {
foreach (FieldInfo fi in o.GetType().GetFields())
{
string value = this[fi.Name]; string value = this[fi.Name];
//this should only be here temporarily, while the new decompression changes are waiting to go into prod. if (value == null) {
if (value == null) } else {
{
//BlueEventLog.logEvent("Error: Metadata missing field " + fi.Name, BlueEventLog2Level.MissingMetadata);
}
else
{
if (fi.FieldType.Equals(typeof(string))) if (fi.FieldType.Equals(typeof(string)))
fi.SetValue(o, value); fi.SetValue(o, value);
else if (fi.FieldType.Equals(typeof(int))) else if (fi.FieldType.Equals(typeof(int)))
@@ -167,11 +139,10 @@ namespace BCG
} }
} }
public object Clone() public object Clone() {
{
MemoryStream ms = new MemoryStream(); MemoryStream ms = new MemoryStream();
this.save(ms); this.save(ms);
ms.Seek(0,SeekOrigin.Begin); ms.Seek(0, SeekOrigin.Begin);
return new MetaData(ms); return new MetaData(ms);
} }
} }
@@ -1,25 +1,25 @@
using System; // using System;
namespace BCG // namespace BCG
{ // {
[Serializable] // [Serializable]
public class RenewMetaData // public class RenewMetaData
{ // {
public string subscriber; // public string subscriber;
public int lease; // public int lease;
public bool c; // public bool c;
public SubAddress getSubAddress()
{return SubAddress.decode(subscriber);}
public RenewMetaData() // public SubAddress getSubAddress()
{} // {return SubAddress.decode(subscriber);}
public RenewMetaData(string s, // public RenewMetaData()
int lease) // {}
{
this.subscriber = s; // public RenewMetaData(string s,
this.lease = lease; // int lease)
} // {
} // this.subscriber = s;
} // this.lease = lease;
// }
// }
// }
@@ -3,37 +3,33 @@ using System.Collections;
using System.Reflection; using System.Reflection;
using BCG.Utilities; using BCG.Utilities;
using System.Collections.Generic; using System.Collections.Generic;
using System.Reactive.Subjects;
namespace BCG namespace BCG {
{ public class TagType2HandlerMap {
public class TagType2HandlerMap
{
private static TagType2HandlerMap myServerHandlerMap = null, private static TagType2HandlerMap myServerHandlerMap = null,
myPublisherStateMap = null; myPublisherStateMap = null;
private Dictionary<string, IPubSubServerTagHandler> myTagTypeToPubHandlerMap = new Dictionary<string, IPubSubServerTagHandler>(); private Dictionary<string, IPubSubServerTagHandler> myTagTypeToPubHandlerMap =
new Dictionary<string, IPubSubServerTagHandler>();
public TagType2HandlerMap() public TagType2HandlerMap() {
{ if (PubSub3Config.instance.PS3_TagTypeToAssemblyMap != null) {
if (PubSub3Config.instance.PS3_TagTypeToAssemblyMap != null) foreach (TagTypeToAssembly.Entry entry in PubSub3Config.instance.PS3_TagTypeToAssemblyMap.entries) {
{ try {
foreach (TagTypeToAssembly.Entry entry in PubSub3Config.instance.PS3_TagTypeToAssemblyMap.entries) var ass_loc = assembly_location;
{ bts.Log.Gen.info($"TagType2HandlerMap entry: {entry} assembly_location: {ass_loc}");
try
{
Assembly assem = Assembly.LoadFrom(entry.assembly); Assembly assem = Assembly.LoadFrom(entry.assembly);
Type t = assem.GetType(entry.handlerClass, true, false); Type t = assem.GetType(entry.handlerClass, true, true);
object obj = Activator.CreateInstance(t); object obj = Activator.CreateInstance(t);
IPubSubServerTagHandler tagHandler = obj as IPubSubServerTagHandler; IPubSubServerTagHandler tagHandler = obj as IPubSubServerTagHandler;
bts.Log.Gen.info($"TagType2HandlerMap successfully created obj: {obj} of type: {t} from assem: {assem}");
string[] tagTypes = tagHandler.supportedTagTypes; string[] tagTypes = tagHandler.supportedTagTypes;
foreach (string tagType in tagTypes) foreach (string tagType in tagTypes) {
{ bts.Log.Gen.info($"TagType2HandlerMap adding tagType: {tagType} tagHandler: {tagHandler}");
myTagTypeToPubHandlerMap[tagType] = tagHandler; myTagTypeToPubHandlerMap[tagType] = tagHandler;
} }
} } catch (Exception ex) {
catch (Exception ex) bts.Log.Gen.error($"TagType2HandlerMap threw exception ex: {ex}");
{
// BlueEventLog.logEvent(string.Format("error happened while loading type from the assembly ", entry.handlerClass, entry.assembly), BlueEventLevel.Warning);
// BlueEventLog.logEvent(ex);
if (ex is OutOfMemoryException) if (ex is OutOfMemoryException)
throw ex; throw ex;
} }
@@ -41,30 +37,41 @@ namespace BCG
} }
} }
private IPubSubServerTagHandler myGetHandlerFromTagType(string tagType) public static string assembly_location {
{ get {
// string codeBase = Assembly.GetExecutingAssembly().CodeBase;
// UriBuilder uri = new UriBuilder(codeBase);
// string path = Uri.UnescapeDataString(uri.Path);
// return Path.GetDirectoryName(path);
// Get the executing assembly
Assembly executingAssembly = Assembly.GetExecutingAssembly();
// Get the location (path) of the executing assembly
string assembly_location = executingAssembly.Location;
return assembly_location;
}
}
private IPubSubServerTagHandler myGetHandlerFromTagType(string tagType) {
IPubSubServerTagHandler rv; IPubSubServerTagHandler rv;
myTagTypeToPubHandlerMap.TryGetValue(tagType, out rv); myTagTypeToPubHandlerMap.TryGetValue(tagType, out rv);
bts.Log.Gen.info($"TagType2HandlerMap.myGetHandlerFromTagType tag_type: {tagType} rv: {rv}");
return rv; return rv;
} }
public static IPubSubServerTagHandler getHandler(string type, bool isServer) public static IPubSubServerTagHandler getHandler(string type, bool isServer) {
{ lock (typeof(IPubSubServerTagHandler)) {
lock (typeof(IPubSubServerTagHandler)) if (isServer) {
{
if (isServer)
{
if (myServerHandlerMap == null) if (myServerHandlerMap == null)
myServerHandlerMap = new TagType2HandlerMap(); myServerHandlerMap = new TagType2HandlerMap();
return myServerHandlerMap.myGetHandlerFromTagType(type); return myServerHandlerMap.myGetHandlerFromTagType(type);
} } else {
else
{
if (myPublisherStateMap == null) if (myPublisherStateMap == null)
myPublisherStateMap = new TagType2HandlerMap(); myPublisherStateMap = new TagType2HandlerMap();
return myPublisherStateMap.myGetHandlerFromTagType(type); return myPublisherStateMap.myGetHandlerFromTagType(type);
} }
} }
} }
} }
} }
@@ -1,17 +1,18 @@
using System; using System;
namespace BCG namespace BCG {
{
[Serializable] [Serializable]
public class TagTypeToAssembly public class TagTypeToAssembly {
{
//[Serializable] //[Serializable]
public class Entry public class Entry {
{ public string handlerClass = "BCG.PubSubServerTagHandler";
public string handlerClass; // TODOds
public string assembly; public string assembly = "C:\\Users\\m_tah\\spark\\build\\Debug\\bin\\net7.0\\DataStreamCore.dll";
public override string ToString() {
return $"handlerClass: {handlerClass} assembly: {assembly}";
}
} }
public Entry[] entries; public Entry[] entries = new Entry[] { new Entry() };
} }
} }
@@ -5,17 +5,13 @@ using System.Text.RegularExpressions;
//using BCG.Utilities.BlueConfiguration; //using BCG.Utilities.BlueConfiguration;
//using BCG.Utilities; //using BCG.Utilities;
namespace BCG namespace BCG {
{
public class PubSub3Config //: ConfigurationRequest public class PubSub3Config //: ConfigurationRequest
{ {
private static PubSub3Config myInstance; private static PubSub3Config myInstance;
public static PubSub3Config instance public static PubSub3Config instance {
{ get {
get if (myInstance == null) {
{
if (myInstance == null)
{
myInstance = new PubSub3Config(); myInstance = new PubSub3Config();
//myInstance = (PubSub3Config) ConfigurationRequester.request(typeof(PubSub3Config)); //myInstance = (PubSub3Config) ConfigurationRequester.request(typeof(PubSub3Config));
} }
@@ -43,7 +39,7 @@ namespace BCG
private string[] myPS3_CrushableTags = new string[] { }; private string[] myPS3_CrushableTags = new string[] { };
//moved from PS2 //moved from PS2
private TagTypeToAssembly myTagTypeToAssemblyMap; private TagTypeToAssembly myTagTypeToAssemblyMap = new TagTypeToAssembly();
private string myPS3_ServerQueue; private string myPS3_ServerQueue;
//substation stuff //substation stuff
@@ -83,122 +79,101 @@ namespace BCG
// bluecomm transition // bluecomm transition
//public ConfigKeys.BlueComm_PSTransition BlueComm_PSTransition { get; set; } //public ConfigKeys.BlueComm_PSTransition BlueComm_PSTransition { get; set; }
public PubSub3Config() public PubSub3Config() {
{
//BlueComm_PSTransition = new BCG.ConfigKeys.BlueComm_PSTransition(true); //BlueComm_PSTransition = new BCG.ConfigKeys.BlueComm_PSTransition(true);
} }
public bool isValid() public bool isValid() {
{
return true; return true;
} }
public int PS3_retriesOnNoBufferSpace public int PS3_retriesOnNoBufferSpace {
{
get { return myPS3_retriesOnNoBufferSpace; } get { return myPS3_retriesOnNoBufferSpace; }
set { myPS3_retriesOnNoBufferSpace = value; } set { myPS3_retriesOnNoBufferSpace = value; }
} }
public int PS3_millisecondsBetweenNoBufferSpaceRetries public int PS3_millisecondsBetweenNoBufferSpaceRetries {
{
get { return myPS3_millisecondsBetweenNoBufferSpaceRetries; } get { return myPS3_millisecondsBetweenNoBufferSpaceRetries; }
set { myPS3_millisecondsBetweenNoBufferSpaceRetries = value; } set { myPS3_millisecondsBetweenNoBufferSpaceRetries = value; }
} }
public int PS3_millisecondsBetweenTimeSliceUpdates public int PS3_millisecondsBetweenTimeSliceUpdates {
{
get { return myPS3_millisecondsBetweenTimeSliceUpdates; } get { return myPS3_millisecondsBetweenTimeSliceUpdates; }
set { myPS3_millisecondsBetweenTimeSliceUpdates = value; } set { myPS3_millisecondsBetweenTimeSliceUpdates = value; }
} }
public bool BlueComm_mustDie public bool BlueComm_mustDie {
{
get { return myBlueComm_mustDie; } get { return myBlueComm_mustDie; }
set { myBlueComm_mustDie = value; } set { myBlueComm_mustDie = value; }
} }
public int PS3_HeartBeatInterval public int PS3_HeartBeatInterval {
{
get { return myPS3_HeartBeatInterval; } get { return myPS3_HeartBeatInterval; }
set { myPS3_HeartBeatInterval = value; } set { myPS3_HeartBeatInterval = value; }
} }
public int PS3_DoItAllSecondsBetweenLogs public int PS3_DoItAllSecondsBetweenLogs {
{
get { return myPS3_DoItAllSecondsBetweenLogs; } get { return myPS3_DoItAllSecondsBetweenLogs; }
set { myPS3_DoItAllSecondsBetweenLogs = value; } set { myPS3_DoItAllSecondsBetweenLogs = value; }
} }
public int PS3_MissingHeartBeatsForGC public int PS3_MissingHeartBeatsForGC {
{
get { return myPS3_MissingHeartBeatsBeforeGC; } get { return myPS3_MissingHeartBeatsBeforeGC; }
set { myPS3_MissingHeartBeatsBeforeGC = value; } set { myPS3_MissingHeartBeatsBeforeGC = value; }
} }
public bool PS3_UseManagedSockets public bool PS3_UseManagedSockets {
{
get { return myPS3_UseManagedSockets; } get { return myPS3_UseManagedSockets; }
set { myPS3_UseManagedSockets = value; } set { myPS3_UseManagedSockets = value; }
} }
public int PS3_StateUpdatePeriod public int PS3_StateUpdatePeriod {
{
get { return myPS3_StateUpdatePeriod; } get { return myPS3_StateUpdatePeriod; }
set { myPS3_StateUpdatePeriod = value; } set { myPS3_StateUpdatePeriod = value; }
} }
public int PS3_StatisticsUpdatePeriod public int PS3_StatisticsUpdatePeriod {
{
get { return myPS3_StatisticsUpdatePeriod; } get { return myPS3_StatisticsUpdatePeriod; }
set { myPS3_StatisticsUpdatePeriod = value; } set { myPS3_StatisticsUpdatePeriod = value; }
} }
public int PS3_DisposedConnectionCheckPeriod public int PS3_DisposedConnectionCheckPeriod {
{
get { return myPS3_DisposedConnectionCheckPeriod; } get { return myPS3_DisposedConnectionCheckPeriod; }
set { myPS3_DisposedConnectionCheckPeriod = value; } set { myPS3_DisposedConnectionCheckPeriod = value; }
} }
public int PS3_MessageProcessingWarningThreshold public int PS3_MessageProcessingWarningThreshold {
{
get { return myPS3_MessageProcessingWarningThreshold; } get { return myPS3_MessageProcessingWarningThreshold; }
set { myPS3_MessageProcessingWarningThreshold = value; } set { myPS3_MessageProcessingWarningThreshold = value; }
} }
public int PS3_millisecondsBetweenSubInfoUpdates public int PS3_millisecondsBetweenSubInfoUpdates {
{
get { return myPS3_millisecondsBetweenSubInfoUpdates; } get { return myPS3_millisecondsBetweenSubInfoUpdates; }
set { myPS3_millisecondsBetweenSubInfoUpdates = value; } set { myPS3_millisecondsBetweenSubInfoUpdates = value; }
} }
public double PS3_LatencyReportThresholdMS public double PS3_LatencyReportThresholdMS {
{
get { return myPS3_LatencyReportThresholdMS; } get { return myPS3_LatencyReportThresholdMS; }
set { myPS3_LatencyReportThresholdMS = value; } set { myPS3_LatencyReportThresholdMS = value; }
} }
public int PS3_ServerPort public int PS3_ServerPort {
{
get { return myPS3_ServerPort; } get { return myPS3_ServerPort; }
set { myPS3_ServerPort = value; } set { myPS3_ServerPort = value; }
} }
public PubSubRoute[] getSubRouting() public PubSubRoute[] getSubRouting() { return getRouting(); }
{ return getRouting(); }
public PubSubRoute[] getPubRouting() public PubSubRoute[] getPubRouting() { return getRouting(); }
{ return getRouting(); }
public PubSubRoute[] getRouting() public PubSubRoute[] getRouting() {
{
// TODO // TODO
return new PubSubRoute[] { }; return new PubSubRoute[] { };
// return getRouting(SitesCore.getMySiteName()); // return getRouting(SitesCore.getMySiteName());
} }
public PubSubRoute[] getRouting(string forSite) public PubSubRoute[] getRouting(string forSite) {
{
// TODO // TODO
return new PubSubRoute[] { }; return new PubSubRoute[] { };
// if (sites_layout != null) { // if (sites_layout != null) {
@@ -214,14 +189,12 @@ namespace BCG
return null; return null;
} }
public List<string> getAllPubSubs() public List<string> getAllPubSubs() {
{
return new List<string>(); return new List<string>();
//return getAllPubSubs(SitesCore.getMySiteName()); //return getAllPubSubs(SitesCore.getMySiteName());
} }
public List<string> getAllPubSubs(string forSite) public List<string> getAllPubSubs(string forSite) {
{
List<string> result = new List<string>(); List<string> result = new List<string>();
foreach (PubSubRoute r in getRouting(forSite)) foreach (PubSubRoute r in getRouting(forSite))
result.AddRange(r.getAllPubSubs()); result.AddRange(r.getAllPubSubs());
@@ -233,17 +206,14 @@ namespace BCG
// public ConfigKeys.sites_siteName sites_siteName { get; set; } // public ConfigKeys.sites_siteName sites_siteName { get; set; }
public TagTypeToAssembly PS3_TagTypeToAssemblyMap public TagTypeToAssembly PS3_TagTypeToAssemblyMap {
{
get { return myTagTypeToAssemblyMap; } get { return myTagTypeToAssemblyMap; }
set { myTagTypeToAssemblyMap = value; } set { myTagTypeToAssemblyMap = value; }
} }
//[Description("PUBSUB: The Message queue to look for pub sub messages")] //[Description("PUBSUB: The Message queue to look for pub sub messages")]
public string PS3_ServerQueue public string PS3_ServerQueue {
{ get {
get
{
if (myPS3_ServerQueue == null || myPS3_ServerQueue == "") if (myPS3_ServerQueue == null || myPS3_ServerQueue == "")
return "PubSubQueue"; return "PubSubQueue";
else else
@@ -252,14 +222,12 @@ namespace BCG
set { myPS3_ServerQueue = value; } set { myPS3_ServerQueue = value; }
} }
public bool PS3_InternallyPublishSubInfo public bool PS3_InternallyPublishSubInfo {
{
get { return myPS3_InternallyPublishSubInfo; } get { return myPS3_InternallyPublishSubInfo; }
set { myPS3_InternallyPublishSubInfo = value; } set { myPS3_InternallyPublishSubInfo = value; }
} }
public string[] PS3_CrushableTags public string[] PS3_CrushableTags {
{
get { return myPS3_CrushableTags; } get { return myPS3_CrushableTags; }
set { myPS3_CrushableTags = value; } set { myPS3_CrushableTags = value; }
} }
@@ -299,95 +267,80 @@ namespace BCG
// } // }
public bool PS3_UseSubstations public bool PS3_UseSubstations {
{
get { return myPS3_UseSubstations; } get { return myPS3_UseSubstations; }
set { myPS3_UseSubstations = value; } set { myPS3_UseSubstations = value; }
} }
public PubSubPortRange PS3_PortRange public PubSubPortRange PS3_PortRange {
{
get { return myPS3_PortRange; } get { return myPS3_PortRange; }
set { myPS3_PortRange = value; } set { myPS3_PortRange = value; }
} }
public bool PS3_BindToRandomPort public bool PS3_BindToRandomPort {
{
get { return myPS3_BindToRandomPort; } get { return myPS3_BindToRandomPort; }
set { myPS3_BindToRandomPort = value; } set { myPS3_BindToRandomPort = value; }
} }
public bool PS3_IsSubstation public bool PS3_IsSubstation {
{
get { return myPS3_IsSubstation; } get { return myPS3_IsSubstation; }
set { myPS3_IsSubstation = value; } set { myPS3_IsSubstation = value; }
} }
public bool PS3_UseDynamicClients public bool PS3_UseDynamicClients {
{
get { return myPS3_UseDynamicClients; } get { return myPS3_UseDynamicClients; }
set { myPS3_UseDynamicClients = value; } set { myPS3_UseDynamicClients = value; }
} }
public bool PS3_UseManagedClientsInNativeCode public bool PS3_UseManagedClientsInNativeCode {
{
get { return myPS3_UseManagedClientsInNativeCode; } get { return myPS3_UseManagedClientsInNativeCode; }
set { myPS3_UseManagedClientsInNativeCode = value; } set { myPS3_UseManagedClientsInNativeCode = value; }
} }
public string PS3_CompressedTagsRegex public string PS3_CompressedTagsRegex {
{
get { return myPS3_CompressedTagsRegex; } get { return myPS3_CompressedTagsRegex; }
set { myPS3_CompressedTagsRegex = value; } set { myPS3_CompressedTagsRegex = value; }
} }
public string PS3_DebugTags public string PS3_DebugTags {
{
get { return myPS3_DebugTags; } get { return myPS3_DebugTags; }
set { myPS3_DebugTags = value; } set { myPS3_DebugTags = value; }
} }
public string PS3_TagsToReportCompressionRatio public string PS3_TagsToReportCompressionRatio {
{
get { return myPS3_TagsToReportCompressionRatio; } get { return myPS3_TagsToReportCompressionRatio; }
set { myPS3_TagsToReportCompressionRatio = value; } set { myPS3_TagsToReportCompressionRatio = value; }
} }
public bool PS3_DisableMSMQ public bool PS3_DisableMSMQ {
{
get { return myPS3_DisableMSMQ; } get { return myPS3_DisableMSMQ; }
set { myPS3_DisableMSMQ = true; } set { myPS3_DisableMSMQ = true; }
} }
public string PS3_IPTranslation public string PS3_IPTranslation {
{
get { return myPS3_IPTranslation; } get { return myPS3_IPTranslation; }
set { myPS3_IPTranslation = value; } set { myPS3_IPTranslation = value; }
} }
public long PS3_GCActivationThreshold public long PS3_GCActivationThreshold {
{
get { return myGCActivationThreshold; } get { return myGCActivationThreshold; }
set { myGCActivationThreshold = value; } set { myGCActivationThreshold = value; }
} }
public int PS3_WaitPeriodAfterGC public int PS3_WaitPeriodAfterGC {
{
get { return myWaitPeriodAfterGC; } get { return myWaitPeriodAfterGC; }
set { myWaitPeriodAfterGC = value; } set { myWaitPeriodAfterGC = value; }
} }
public bool PS3_UsePrepublishedServerState public bool PS3_UsePrepublishedServerState {
{
get { return myPS3_UsePrepublishedServerState; } get { return myPS3_UsePrepublishedServerState; }
set { myPS3_UsePrepublishedServerState = value; } set { myPS3_UsePrepublishedServerState = value; }
} }
public int PS3_MaxPacketSize public int PS3_MaxPacketSize {
{
get { return myPS3_MaxPacketSize; } get { return myPS3_MaxPacketSize; }
set { myPS3_MaxPacketSize = value; } set { myPS3_MaxPacketSize = value; }
} }
@@ -404,21 +357,17 @@ namespace BCG
// set { myRecovery_PubSubCopies = value; } // set { myRecovery_PubSubCopies = value; }
// } // }
public bool isTagDebugTag(string tag) public bool isTagDebugTag(string tag) {
{ if (myDebugTagsRegex == null) {
if (myDebugTagsRegex == null)
{
myDebugTagsRegex = new Regex(myPS3_DebugTags, RegexOptions.IgnoreCase); myDebugTagsRegex = new Regex(myPS3_DebugTags, RegexOptions.IgnoreCase);
} }
return myDebugTagsRegex.IsMatch(tag); return myDebugTagsRegex.IsMatch(tag);
} }
public bool isTagCompressed(string tag) public bool isTagCompressed(string tag) {
{
if (myCompressedTagsRegex == null) if (myCompressedTagsRegex == null) {
{
myCompressedTagsRegex = new Regex(myPS3_CompressedTagsRegex, RegexOptions.IgnoreCase); myCompressedTagsRegex = new Regex(myPS3_CompressedTagsRegex, RegexOptions.IgnoreCase);
} }
@@ -426,10 +375,8 @@ namespace BCG
return (myCompressedTagsRegex.IsMatch(tag)); return (myCompressedTagsRegex.IsMatch(tag));
} }
public bool shouldReportTagsCompressionRatio(string tag) public bool shouldReportTagsCompressionRatio(string tag) {
{ if (myTagsToReportCompressionRatioRegex == null) {
if (myTagsToReportCompressionRatioRegex == null)
{
myTagsToReportCompressionRatioRegex = new Regex(myPS3_TagsToReportCompressionRatio, RegexOptions.IgnoreCase); myTagsToReportCompressionRatioRegex = new Regex(myPS3_TagsToReportCompressionRatio, RegexOptions.IgnoreCase);
} }
return (myTagsToReportCompressionRatioRegex.IsMatch(tag)); return (myTagsToReportCompressionRatioRegex.IsMatch(tag));
@@ -45,8 +45,7 @@ namespace BCG
public SendConfirmationDelegate onSendDel; public SendConfirmationDelegate onSendDel;
public AsyncMessage(byte[] theBuffer, int theOffset) public AsyncMessage(byte[] theBuffer, int theOffset)
: this(theBuffer, theOffset, String.Empty, null) : this(theBuffer, theOffset, String.Empty, null) { }
{ }
public AsyncMessage(byte[] theBuffer, int theOffset, string theMessageId, SendConfirmationDelegate theOnSendDel) public AsyncMessage(byte[] theBuffer, int theOffset, string theMessageId, SendConfirmationDelegate theOnSendDel)
{ {
@@ -445,7 +444,7 @@ namespace BCG
myDataReceived = true; myDataReceived = true;
Console.WriteLine($"AsyncTCPConnection Data received for the first time from peer {myPeer.ToString()}"); Console.WriteLine($"AsyncTCPConnection Data received for the first time from peer {myPeer.ToString()}");
} }
myMessageHandler.onData(new RemoteMessage(data, RemoteMessageType.data, myPeer)); myMessageHandler.onData(new RemoteMessage(data, RemoteMessageType.Data, myPeer));
} }
else if (msgType == 1) else if (msgType == 1)
@@ -456,7 +455,7 @@ namespace BCG
Console.WriteLine($"AsyncTCPConnection HB received for the first time from peer {myPeer.ToString()}"); Console.WriteLine($"AsyncTCPConnection HB received for the first time from peer {myPeer.ToString()}");
} }
myLastHBReceivedTime = DateTime.Now; myLastHBReceivedTime = DateTime.Now;
myMessageHandler.onData(new RemoteMessage(null, RemoteMessageType.heartbeat, myPeer)); myMessageHandler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, myPeer));
} }
myReadSize(); myReadSize();
} }
@@ -0,0 +1,296 @@
using BCG;
using Google.Protobuf;
using System;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using BCG;
using System.Xml.Serialization;
using System.Reflection;
using BCG.Utilities.General;
using System.Net.WebSockets;
namespace BCG
{
public class AsyncTCPConnection2 : ITCPConnection, IWorker, IDisposable
{
//private Socket _socket;
// tied to the socket above
private NetworkStream _nw_stream;
private readonly ConnectionSide _side;
ITCPMessageHandler _message_handler;
private int _msg_id = 0;
RemoteConnectionInfo _peer_address;
RemoteConnectionInfo _self_address;
private TimeSpan _hb_period = TimeSpan.FromSeconds(1);
private DateTime _last_hb_sent = DateTime.MinValue;
private DateTime _last_hb_received = DateTime.MaxValue;
private static byte _msg_type_data = 1;
private static byte _msg_type_hb = 0;
private bool _is_disposed = false;
SyncQueue<QueuedMessage> _data_send_queue = new SyncQueue<QueuedMessage>();
public class QueuedMessage
{
public string id;
public SendConfirmationDelegate del;
public byte[] data;
public QueuedMessage(string id, SendConfirmationDelegate del, byte[] data)
{
id = id;
del = del;
data = data;
}
public QueuedMessage(byte[] data)
: this(String.Empty, null, data) { }
}
public enum ConnectionSide
{
Client,
Server
}
public AsyncTCPConnection2(Socket socket, ConnectionSide side)
{
//_socket = socket;
_nw_stream = new NetworkStream(socket);
_side = side;
IPEndPoint r_ep = socket.RemoteEndPoint as IPEndPoint;
_peer_address = new RemoteConnectionInfo(r_ep.Address.ToString(), r_ep.Port);
var loc_ep = socket.LocalEndPoint as IPEndPoint;
_self_address = new RemoteConnectionInfo(loc_ep.Address.ToString(), loc_ep.Port);
}
public void register(ITCPMessageHandler message_handler)
{
_message_handler = message_handler;
Task.Run(async () => _keep_reading());
Task.Run(async () => await _keep_sending());
//_keep_reading();
// if add a _keep_sending threadis added then we can maek send a non async
// methid making it much easier to integrate with the rest
}
// this thread pulls
private async Task _keep_sending()
{
bts.Log.Gen.info($"TCPConnection._keep_sending starting");
while (!_is_disposed)
{
try
{
QueuedMessage m = _data_send_queue.dequeue(100);
if (m == null)
{
continue;
}
await send_async(m.data);
if (String.IsNullOrEmpty(m.id))
{
m.del(m.id, true);
}
}
catch (Exception ex)
{
bts.Log.Gen.error($"TCPConnection._keep_sending ex: {ex.Message}");
await Task.Delay(100);
_handle_error(ex);
}
}
}
private async Task _keep_reading()
{
bts.Log.Gen.info($"TCPConnection._keep_reading starting");
while (!_is_disposed)
{
try
{
byte[] msg = await TCPSocketUtils.read_bytes_async(_nw_stream);
bts.Log.Gen.info($"AsyncTCPConnection2._keep_reading side: {_side} self: {get_self_address()} peer: {get_peer_address()} msg len: {msg.Length}");
await _handle_msg(msg);
}
catch (Exception ex)
{
bts.Log.Gen.error($"TCPConnection._keep_reading ex: {ex.Message}");
await Task.Delay(100);
_handle_error(ex);
}
}
}
private async Task _handle_msg(byte[] msg)
{
// read the first byte to read the msg type and deal with it accordingly
BinaryReader br = new BinaryReader(new MemoryStream(msg));
byte msg_type = br.ReadByte();
if (msg_type == _msg_type_data)
{
byte[] data = br.ReadBytes(msg.Length - 1);
RemoteMessage m = new RemoteMessage(data, RemoteMessageType.Data, _peer_address);
_message_handler.onData(m);
}
else
{
RemoteMessage m = new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_address);
_message_handler.onData(m);
}
}
// not very efficient
private static byte[] _prefix_msg_type(byte[] data)
{
MemoryStream ms = new MemoryStream();
BinaryWriter bw = new BinaryWriter(ms);
bw.Write(_msg_type_data);
bw.Write(data);
ms.Position = 0;
return ms.ToArray();
}
private async void _handle_error(Exception ex)
{
bts.Log.Gen.error($"TCPConnection._handle_error ex: {ex}");
try
{
var socket = _nw_stream.Socket;
socket.Close();
socket.Dispose();
}
catch (Exception ex2)
{
bts.Log.Gen.error($"TCPConnection._handle_error got exception when closing Socket ex: {ex}");
}
_is_disposed = true;
RemoteMessage rm = new RemoteMessage(new byte[] { }, RemoteMessageType.Disconnection, _peer_address);
_message_handler.onData(rm);
}
public void send_with_confirmation(byte[] data, string confId, SendConfirmationDelegate del) { }
public async Task send_with_confirmation_async(byte[] data, string conf_id, SendConfirmationDelegate del)
{
try
{
conf_id = (++_msg_id).ToString();
await TCPSocketUtils.send_bytes_async(_nw_stream, data);
del(conf_id, true);
}
catch (Exception ex)
{
_handle_error(ex);
}
}
public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del)
{
conf_id = (++_msg_id).ToString();
send(data);
del(conf_id, true);
}
public void send(byte[] data)
{
Task.Run(async () => await send_async(data));
bts.Log.Gen.info($"AsyncTCPConnection2.send self: {get_self_address()} peer: {get_peer_address()} msg len: {data.Length} sent successfully");
//_data_send_queue.enqueue(new QueuedMessage(data));
}
public async Task send_async(byte[] data)
{
// prefix 4 bytes for type (data vs hb)
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
bw.Write(_msg_type_data);
bw.Write(data);
try
{
await TCPSocketUtils.send_bytes_async(_nw_stream, str.ToArray());
}
catch (Exception ex)
{
bts.Log.Gen.error($"AsyncTCPConnection2.send_async ex: {ex}");
_handle_error(ex);
}
}
public void getPeerAddress(out string ip, out int port)
{
ip = _peer_address.ip;
port = _peer_address.port;
///.Net has some problems getting the port, so using
///native code for that
//sockaddr addr = new sockaddr();
//int namelen = 16;
//getsockname(mySock.Handle, ref addr, ref namelen);
//port = addr.sin_port;
}
public RemoteConnectionInfo get_peer_address()
{
string ip; int port;
getPeerAddress(out ip, out port);
return new RemoteConnectionInfo(ip, port);
}
public RemoteConnectionInfo get_self_address()
{
string ip; int port;
getSelfAddress(out ip, out port);
return new RemoteConnectionInfo(ip, port);
}
public void getSelfAddress(out string ip, out int port)
{
ip = _self_address.ip;
port = _self_address.port;
}
private async void _send_hb()
{
if (_last_hb_sent - DateTime.Now > _hb_period)
{
MemoryStream ms = new MemoryStream();
BinaryWriter bw = new BinaryWriter(ms);
bw.Write(_msg_type_hb);
ms.Position = 0;
byte[] hb_msg = ms.ToArray();
await TCPSocketUtils.send_bytes_async(_nw_stream, hb_msg);
}
}
public void doWork()
{
_send_hb();
}
public void Dispose()
{
_nw_stream.Close();
_nw_stream.Dispose();
_is_disposed = true;
}
public bool isDisposed
{
get
{
return _is_disposed;
}
}
}
}
@@ -4,21 +4,19 @@ using System.Text;
using System.Net; using System.Net;
using System.Net.Sockets; using System.Net.Sockets;
using System.Threading; using System.Threading;
using System.Threading.Tasks;
//using BCG.Utilities; //using BCG.Utilities;
//using BCG.Utilities.General; //using BCG.Utilities.General;
namespace BCG namespace BCG {
{ internal class SocketConnectionArgs {
internal class SocketConnectionArgs
{
public EndPoint ep; public EndPoint ep;
public Socket sock; public Socket sock;
public ITCPConnectionHandler handler; public ITCPConnectionHandler handler;
public string serverURL; public string serverURL;
internal SocketConnectionArgs(EndPoint theEp, Socket theSock, ITCPConnectionHandler theHandler, internal SocketConnectionArgs(EndPoint theEp, Socket theSock, ITCPConnectionHandler theHandler,
string theServerURL) string theServerURL) {
{
ep = theEp; ep = theEp;
sock = theSock; sock = theSock;
handler = theHandler; handler = theHandler;
@@ -26,46 +24,46 @@ namespace BCG
} }
} }
public class ClientToServerManagedTCPConnector public class ClientToServerManagedTCPConnector {
{
private static Random myRnd = new Random(DateTime.Now.Millisecond); private static Random myRnd = new Random(DateTime.Now.Millisecond);
private static TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval); private static TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval);
private static int myWaitTimeBeforeRetryingUpperLimit = 5000, private static int myWaitTimeBeforeRetryingUpperLimit = 5000,
myWaitTimeBeforeRetryingLowerLimit = 1000; myWaitTimeBeforeRetryingLowerLimit = 1000;
public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler) public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler) {
{
// TODOds
// BlueEventLog.logEvent(String.Format("Connection being attempted to server {0}", serverAddress.ToString()),
// PS3LogLevels.connection);
Console.WriteLine($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}"); Console.WriteLine($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
EndPoint ep = new IPEndPoint(IPAddress.Parse(serverAddress.ip), serverAddress.port); EndPoint ep = new IPEndPoint(IPAddress.Parse(serverAddress.ip), serverAddress.port);
sock.BeginConnect(ep, new AsyncCallback(myOnConnect), new SocketConnectionArgs(ep, sock, handler, serverAddress.ToString())); sock.BeginConnect(ep, new AsyncCallback(myOnConnect), new SocketConnectionArgs(ep, sock, handler, serverAddress.ToString()));
} }
private static void myOnConnect(IAsyncResult res) public static async Task connect_to_async(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
{ bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
}
public static void connect_to(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) {
bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}");
Task.Run(async () => {
await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts);
});
}
private static void myOnConnect(IAsyncResult res) {
Console.WriteLine("ClientToServerManagedTCPConnector.myOnConnect " + res.ToString() + " " + res.AsyncState.ToString()); Console.WriteLine("ClientToServerManagedTCPConnector.myOnConnect " + res.ToString() + " " + res.AsyncState.ToString());
SocketConnectionArgs args = res.AsyncState as SocketConnectionArgs; SocketConnectionArgs args = res.AsyncState as SocketConnectionArgs;
try try {
{
args.sock.EndConnect(res); args.sock.EndConnect(res);
} } catch (SocketException sex) {
catch (SocketException sex)
{
string errMsg = String.Format("{0}: Connection to {1} failed.", DateTime.Now.ToString(), args.serverURL.ToString()); string errMsg = String.Format("{0}: Connection to {1} failed.", DateTime.Now.ToString(), args.serverURL.ToString());
//BlueEventLog.logEvent(sex.Message + ";" + errMsg, PS3LogLevels.disconnection); //BlueEventLog.logEvent(sex.Message + ";" + errMsg, PS3LogLevels.disconnection);
//Console.WriteLine(errMsg); //Console.WriteLine(errMsg);
if (args.handler.isDisposed) if (args.handler.isDisposed) {
{
string doneMsg = String.Format("{0}: Connection to {1} was disposed. Not Retrying.", DateTime.Now.ToString(), args.serverURL.ToString()); string doneMsg = String.Format("{0}: Connection to {1} was disposed. Not Retrying.", DateTime.Now.ToString(), args.serverURL.ToString());
//Console.WriteLine(doneMsg); //Console.WriteLine(doneMsg);
// BlueEventLog.logEvent(doneMsg, PS3LogLevels.disconnection); // BlueEventLog.logEvent(doneMsg, PS3LogLevels.disconnection);
return; return;
} } else {
else
{
string retryMsg = String.Format("{0}: Retrying Connection to {1}.", DateTime.Now.ToString(), args.serverURL.ToString()); string retryMsg = String.Format("{0}: Retrying Connection to {1}.", DateTime.Now.ToString(), args.serverURL.ToString());
//Console.WriteLine(retryMsg); //Console.WriteLine(retryMsg);
//BlueEventLog.logEvent(retryMsg, PS3LogLevels.disconnection); //BlueEventLog.logEvent(retryMsg, PS3LogLevels.disconnection);
@@ -2,15 +2,13 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Text; using System.Text;
namespace BCG namespace BCG {
{
public delegate void SendConfirmationDelegate(string messageId, bool confirmed); public delegate void SendConfirmationDelegate(string messageId, bool confirmed);
/// <summary> /// <summary>
/// This is the very basic connection that sits right /// This is the very basic connection that sits right
/// on top of the sockets (unmanaged, managed whatever). /// on top of the sockets (used on both server and client)
/// </summary> /// </summary>
public interface ITCPConnection : IDisposable public interface ITCPConnection : IDisposable {
{
/// <summary> /// <summary>
/// The delegate is fired signalling if the data send was /// The delegate is fired signalling if the data send was
/// successful or not /// successful or not
@@ -18,7 +16,7 @@ namespace BCG
/// <param name="data"></param> /// <param name="data"></param>
/// <param name="confId"></param> /// <param name="confId"></param>
/// <param name="del"></param> /// <param name="del"></param>
void sendWithConfirmation (byte[] data, ref string confId, SendConfirmationDelegate del); void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del);
/// <summary> /// <summary>
/// sends data to the other side. theres no guaranatee that /// sends data to the other side. theres no guaranatee that
/// data is delievered. /// data is delievered.
@@ -54,6 +52,6 @@ namespace BCG
/// hence this is an indicator of whether the connection is /// hence this is an indicator of whether the connection is
/// alive or dead /// alive or dead
/// </summary> /// </summary>
bool isDisposed { get;} bool isDisposed { get; }
} }
} }
@@ -2,46 +2,51 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Text; using System.Text;
using System.Net; using System.Net;
using bts.utils;
namespace BCG namespace BCG {
{ public class RemoteConnectionInfo {
public class RemoteConnectionInfo
{
private string myIp; private string myIp;
private int myPort; private int myPort;
private IPEndPoint myEndPoint; private IPEndPoint myEndPoint;
public RemoteConnectionInfo(string ip, int port) public RemoteConnectionInfo(string ip, int port) {
{
myIp = ip; myIp = ip;
myPort = port; myPort = port;
} }
public RemoteConnectionInfo(string ip, short port, IPEndPoint endPoint) public RemoteConnectionInfo(string ip, short port, IPEndPoint endPoint)
: this (ip, port) : this(ip, port) {
{
myEndPoint = endPoint; myEndPoint = endPoint;
} }
public string ip public string ip {
{
get { return myIp; } get { return myIp; }
} }
public int port public int port {
{
get { return myPort; } get { return myPort; }
} }
public IPEndPoint endPoint public IPEndPoint endPoint {
{
get { return myEndPoint; } get { return myEndPoint; }
} }
public override string ToString() public override string ToString() {
{
return ip + ":" + port; return ip + ":" + port;
} }
public override int GetHashCode() {
return ip.GetHashCode() ^ port.GetHashCode();
}
public override bool Equals(object? obj) {
if (obj == null) {
return false;
}
RemoteConnectionInfo other = obj as RemoteConnectionInfo;
return other != null && other.ip == ip && other.port == port;
}
} }
} }
@@ -9,7 +9,7 @@ namespace BCG
/// </summary> /// </summary>
public enum RemoteMessageType public enum RemoteMessageType
{ {
heartbeat, data, connection, disconnection Heartbeat, Data, Connection, Disconnection
} }
/// <summary> /// <summary>
@@ -45,5 +45,11 @@ namespace BCG
get { return myConnInfo; } get { return myConnInfo; }
} }
override public string ToString()
{
Func<byte[], int> len = (byte[] m) => { return m == null ? 0 : m.Length; };
return $"type: {myMsgType} conn: {myConnInfo} msg: {len(myMsg)}";
}
} }
} }
@@ -5,39 +5,30 @@ using System.Text;
//using BCG.Utilities.General; //using BCG.Utilities.General;
//using BCG.Utilities; //using BCG.Utilities;
namespace BCG namespace BCG {
{ public class ServerToClientManagedTCPConnectionAcceptor {
public class ServerToClientManagedTCPConnectionAcceptor
{
private static Dictionary<int, SingleServerToClientManagedTCPConnectionAcceptor> myPortToSingleServerAcceptorMap = new Dictionary<int, SingleServerToClientManagedTCPConnectionAcceptor>(); private static Dictionary<int, SingleServerToClientManagedTCPConnectionAcceptor> myPortToSingleServerAcceptorMap = new Dictionary<int, SingleServerToClientManagedTCPConnectionAcceptor>();
private static object mySyncObject = new object(); private static object mySyncObject = new object();
public static int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan) public static int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan) {
{
SingleServerToClientManagedTCPConnectionAcceptor theAcceptor = new SingleServerToClientManagedTCPConnectionAcceptor(); SingleServerToClientManagedTCPConnectionAcceptor theAcceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
int portNo = theAcceptor.activateWithRandomPort(handler, clientHBTimeSpan); int portNo = theAcceptor.activateWithRandomPort(handler, clientHBTimeSpan);
lock (mySyncObject) lock (mySyncObject) {
{
myPortToSingleServerAcceptorMap[portNo] = theAcceptor; myPortToSingleServerAcceptorMap[portNo] = theAcceptor;
} }
return portNo; return portNo;
} }
public static int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) public static int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) {
{ lock (mySyncObject) {
lock (mySyncObject) if (myPortToSingleServerAcceptorMap.ContainsKey(port)) {
{
if (myPortToSingleServerAcceptorMap.ContainsKey(port))
{
string msg = string.Format("Error: Application is already accepting clients on port {0}", port); string msg = string.Format("Error: Application is already accepting clients on port {0}", port);
// BlueEventLog.logEvent(msg, BlueEventLog2Level.Error); // BlueEventLog.logEvent(msg, BlueEventLog2Level.Error);
return port; return port;
} } else {
else
{
SingleServerToClientManagedTCPConnectionAcceptor theAcceptor = new SingleServerToClientManagedTCPConnectionAcceptor(); SingleServerToClientManagedTCPConnectionAcceptor theAcceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
int assignedport = theAcceptor.activate(handler, port, clientHBTimeSpan); int assignedport = theAcceptor.activate(handler, port, clientHBTimeSpan);
myPortToSingleServerAcceptorMap[assignedport] = theAcceptor; myPortToSingleServerAcceptorMap[assignedport] = theAcceptor;
@@ -46,6 +37,23 @@ namespace BCG
} //end lock } //end lock
} //end activate code } //end activate code
public static int accept_connections(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) {
lock (mySyncObject) {
if (myPortToSingleServerAcceptorMap.ContainsKey(port)) {
string msg = string.Format("Error: Application is already accepting clients on port {0}", port);
// BlueEventLog.logEvent(msg, BlueEventLog2Level.Error);
return port;
} else {
SingleServerToClientManagedTCPConnectionAcceptor ps_acceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
int assignedport = ps_acceptor.accept_on_port(handler, "127.0.0.1", port, clientHBTimeSpan);
myPortToSingleServerAcceptorMap[assignedport] = ps_acceptor;
return assignedport;
}
} //end lock
} //end activate code
} }
} }
@@ -4,13 +4,12 @@ using System.Text;
using System.Threading; using System.Threading;
using System.Net; using System.Net;
using System.Net.Sockets; using System.Net.Sockets;
using System.Threading.Tasks;
//using BCG.Utilities; //using BCG.Utilities;
namespace BCG namespace BCG {
{ public class SingleServerToClientManagedTCPConnectionAcceptor {
public class SingleServerToClientManagedTCPConnectionAcceptor
{
private TimeSpan myHBTimeSpan, private TimeSpan myHBTimeSpan,
myWaitTimeBeforeRetrying = new TimeSpan(0, 0, 0, 0, 250); myWaitTimeBeforeRetrying = new TimeSpan(0, 0, 0, 0, 250);
private ITCPConnectionHandler myHandler; private ITCPConnectionHandler myHandler;
@@ -18,58 +17,50 @@ namespace BCG
private int myPort; private int myPort;
public int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan) public int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan) {
{
myHBTimeSpan = clientHBTimeSpan; myHBTimeSpan = clientHBTimeSpan;
myHandler = handler; myHandler = handler;
myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
int port = 0; int port = 0;
bool madeItThrough = false; bool madeItThrough = false;
// BlueEventLog.logEvent("Looking for open port to run the server on.", PS3LogLevels.connection); // BlueEventLog.logEvent("Looking for open port to run the server on.", PS3LogLevels.connection);
Random randomizer = new Random(); Random randomizer = new Random();
PubSubPortRange theRange = PubSub3Config.instance.PS3_PortRange; PubSubPortRange theRange = PubSub3Config.instance.PS3_PortRange;
//if the range of ports is 1000, then trying 5,000 times means //if the range of ports is 1000, then trying 5,000 times means
//your odds of missing an open port are less than 1% //your odds of missing an open port are less than 1%
int maxAttempts = (theRange.maxPort - theRange.minPort) * 5; int maxAttempts = (theRange.maxPort - theRange.minPort) * 5;
int totalAttempts = 0; int totalAttempts = 0;
while (!madeItThrough) while (!madeItThrough) {
{ port = randomizer.Next();
port = randomizer.Next();
port = theRange.minPort + (port % (theRange.maxPort - theRange.minPort)); port = theRange.minPort + (port % (theRange.maxPort - theRange.minPort));
try try {
{
myAcceptSock.Bind(new IPEndPoint(IPAddress.Any, port)); myAcceptSock.Bind(new IPEndPoint(IPAddress.Any, port));
madeItThrough = true; madeItThrough = true;
} } catch (Exception) {
catch (Exception)
{
totalAttempts++; totalAttempts++;
if (totalAttempts < maxAttempts){ if (totalAttempts < maxAttempts) {
//BlueEventLog.logEvent("Couldn't bind to port " + port.ToString() + "; retrying...", PS3LogLevels.connection); //BlueEventLog.logEvent("Couldn't bind to port " + port.ToString() + "; retrying...", PS3LogLevels.connection);
} } else {
else
{
// BlueEventLog2.instance().abort("Couldn't find an open port to advertise the server on. Consider modifying PS3_PortRange. Talk to Mark for details."); // BlueEventLog2.instance().abort("Couldn't find an open port to advertise the server on. Consider modifying PS3_PortRange. Talk to Mark for details.");
} }
} }
} }
myPort = port; myPort = port;
myAcceptSock.Listen(250); myAcceptSock.Listen(250);
myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
return myPort; return myPort;
} }
public int activate (ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) public int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) {
{
myHBTimeSpan = clientHBTimeSpan; myHBTimeSpan = clientHBTimeSpan;
myHandler = handler; myHandler = handler;
myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
IPEndPoint endpoint = new IPEndPoint (IPAddress.Any, port); IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, port);
myAcceptSock.Bind(endpoint); myAcceptSock.Bind(endpoint);
myAcceptSock.Listen(250); myAcceptSock.Listen(250);
myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
@@ -77,15 +68,18 @@ namespace BCG
return myPort; // not sure why we need this class variable myPort, but I am just keeping it up with the existing code return myPort; // not sure why we need this class variable myPort, but I am just keeping it up with the existing code
} }
private void myOnAccept(IAsyncResult res) public int accept_on_port(ITCPConnectionHandler handler, string ip, int port, TimeSpan client_HB_timespan) {
{ Task.Run(async () => {
await TCPSocketUtils.accept_connections_async(ip, port, handler, TimeSpan.FromSeconds(1));
});
return port;
}
private void myOnAccept(IAsyncResult res) {
Socket newSock; Socket newSock;
try try {
{
newSock = myAcceptSock.EndAccept(res); newSock = myAcceptSock.EndAccept(res);
} } catch (Exception ex) {
catch (Exception ex)
{
// BlueEventLog.logEvent(ex); // BlueEventLog.logEvent(ex);
Thread.Sleep(myWaitTimeBeforeRetrying); Thread.Sleep(myWaitTimeBeforeRetrying);
myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
@@ -98,10 +92,10 @@ namespace BCG
ITCPConnection newConn = new AsyncTCPConnection(newSock, myHBTimeSpan); ITCPConnection newConn = new AsyncTCPConnection(newSock, myHBTimeSpan);
myHandler.onConnection(newConn); myHandler.onConnection(newConn);
myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { });
string ip;int port; string ip; int port;
newConn.getPeerAddress (out ip, out port); newConn.getPeerAddress(out ip, out port);
string connMsg = String.Format("Connection to port {2} made from {0}:{1}", ip, port, myPort); string connMsg = String.Format("Connection to port {2} made from {0}:{1}", ip, port, myPort);
// BlueEventLog.logEvent(connMsg, PS3LogLevels.connection); // BlueEventLog.logEvent(connMsg, PS3LogLevels.connection);
//Console.WriteLine(connMsg); //Console.WriteLine(connMsg);
} }
} }
@@ -0,0 +1,113 @@
using System;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using BCG;
using System.Xml.Serialization;
using System.Reflection;
using BCG.Utilities.General;
using System.Net.WebSockets;
namespace BCG
{
// utility functions for TCP socket async communications. Does the following
// client to server connect
// server to client connect accepting
// send data
// receive data
public static class TCPSocketUtils
{
#region socket opening and connecting
// keeps accepting connections forever
public static async Task accept_connections_async(string ip, int port, ITCPConnectionHandler handler,
TimeSpan wait_after_error)
{
IPAddress ipAddress = String.IsNullOrEmpty(ip) ? IPAddress.Parse(ip) : IPAddress.Any;
var ip_end_point = new IPEndPoint(ipAddress, port);
Socket listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
listener.Bind(ip_end_point);
listener.Listen(10);
Socket client_socket = null;
while (true)
{
try
{
// keeps on accepting connections
client_socket = await listener.AcceptAsync();
AsyncTCPConnection2 new_conn = new(client_socket, AsyncTCPConnection2.ConnectionSide.Server);
handler.onConnection(new_conn);
}
catch (Exception ex)
{
bts.Log.Gen.error($"TCPConnection.accept ex: {ex} trying again");
Task.Delay(wait_after_error);
}
}
}
// connects to the server and ona successful connection, calls con__handler.on_connection_async with an ITCPConnection.
// In case of failures to connect it waits for a given amount of time before retrying
public static async Task<Socket> connect_to_server_async(string ip, int server_port, ITCPConnectionHandler conn_handler,
TimeSpan wait_after_failed_attempt, int n_max_attempts = 1024)
{
Socket client_socket = null;
for (int i = 0; i < n_max_attempts; i++)
{
try
{
string log_conn = $"{ip}:{server_port}";
bts.Log.Gen.info($"AsyncTCPConnection2.connect_to_server conn: {log_conn} attempt no: {i}");
IPAddress server_ip_address = IPAddress.Parse(ip);
client_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await client_socket.ConnectAsync(server_ip_address, server_port);
bts.Log.Gen.info($"AsyncTCPConnection2.connect_to_server conn: {log_conn} successfully connected after {i} attempts");
ITCPConnection new_conn = new AsyncTCPConnection2(client_socket, AsyncTCPConnection2.ConnectionSide.Client);
conn_handler.onConnection(new_conn);
break;
}
catch (Exception ex)
{
bts.Log.Gen.error($"AsyncTCPConnection2.connect_to_server threw ex: {ex}, will reattempt now");
Task.Delay(wait_after_failed_attempt);
}
}
return client_socket;
}
#endregion
#region data sending/receiving
// prefixes the size of the data and sends it as 2 sends, for size and data
// (can be done as one send perhaps)
public static async Task send_bytes_async(NetworkStream nw_stream, byte[] data_buff)
{
var data_sz = data_buff.Length;
byte[] sz_buff = new byte[4];
// aend the size first
BinaryWriter bw = new BinaryWriter(new MemoryStream(sz_buff));
bw.Write(data_sz);
await nw_stream.WriteAsync(sz_buff, 0, 4);
await nw_stream.WriteAsync(data_buff, 0, data_buff.Length);
}
// reads the bytes on a connected socket, by doing the opposite of the above,
// reads the size first and then the data
public static async Task<byte[]> read_bytes_async(NetworkStream nw_stream)
{
byte[] sz_buffer = new byte[4];
//Asynchronously reads a sequence of bytes from the current stream and advances the position within the stream by the number of bytes read.
int bytesRead = await nw_stream.ReadAsync(sz_buffer, 0, sz_buffer.Length);
BinaryReader br = new BinaryReader(new MemoryStream(sz_buffer));
int msg_sz = br.ReadInt32();
byte[] data_buffer = new byte[msg_sz];
bytesRead = await nw_stream.ReadAsync(data_buffer, 0, data_buffer.Length);
//Console.WriteLine($"read_bytes sz: {data_buffer.Length}");
return data_buffer;
}
#endregion
}
}
+23 -15
View File
@@ -4,31 +4,39 @@ using System.IO;
using BCG; using BCG;
using BCG.Utilities; using BCG.Utilities;
namespace BCG namespace BCG {
{ // PubItems are published to the pubsub wrapped in this class. It contains PubMetaData, wrapped in the MetaData
// Tuple of // object and the actualy published payload in the body stream
public class PubSubMessage public class PubSubMessage {
{ public PubSubMessage(MetaData metaData, Stream body) { this.meta_data = metaData; this.body = body; }
public PubSubMessage (MetaData metaData, Stream body)
{this.metaData = metaData; this.body = body;}
public PubSubMessage (MetaData metaData) public PubSubMessage(PubMetaData pub_meta_data, Stream body)
{this.metaData = metaData;} : this(new MetaData(pub_meta_data), body) { }
public MetaData metaData; public PubSubMessage(MetaData metaData) { this.meta_data = metaData; }
public MetaData meta_data;
public Stream body = new MemoryStream(0); public Stream body = new MemoryStream(0);
public Stream toStream() public Stream to_stream() {
{
MemoryStream ms = new MemoryStream(); MemoryStream ms = new MemoryStream();
metaData.save(ms); meta_data.save(ms);
if (body.Length == 0 || body.Length == body.Position) if (body.Length == 0 || body.Length == body.Position) {
bts.Log.Gen.error("Error: Trying to publishing message with no content.");
Console.WriteLine("Error: Trying to publishing message with no content."); Console.WriteLine("Error: Trying to publishing message with no content.");
}
StreamUtilityMethods.copyTo(new Stream[] { body }, ms); StreamUtilityMethods.copyTo(new Stream[] { body }, ms);
ms.Position = 0; ms.Position = 0;
return ms; return ms;
} }
public byte[] to_bytes() {
return StreamUtilityMethods.getBytesFromStream(to_stream());
}
public override string ToString() {
return $"PubSubMessage meta:data: {meta_data.ToString()} body.Posiiton: {body.Position} body.Length: {body.Length}";
}
} }
} }
+7 -8
View File
@@ -2,22 +2,21 @@ using System;
using System.Collections; using System.Collections;
using System.IO; using System.IO;
namespace BCG namespace BCG {
{
/// <summary> /// <summary>
/// this is the data returned from the /// this is the data returned from the
/// pubsub server /// pubsub server
/// </summary> /// </summary>
public class PubSubUpdate public class PubSubUpdate {
{
public PubSubUpdate(string tag, string tagType, Stream body) { this.tag = tag; this.body = body; this.tagType = tagType; }
public PubSubUpdate(string tag, string tagType, Stream body)
{ this.tag = tag; this.body = body; this.tagType = tagType; }
public string tag; public string tag;
public Stream body; public Stream body;
public string tagType; public string tagType;
public override string ToString() {
return $"PubSubUpdate tag: {tag} tag_type: {tagType} len: {body.Position}";
}
}; };
} }
+26 -130
View File
@@ -7,171 +7,67 @@ using System.Text.RegularExpressions;
using BCG.Utilities; using BCG.Utilities;
namespace BCG namespace BCG {
{ public class PublisherCore : IPublisherCore {
public class PublisherCore : IPublisherCore
{
//private static BlueCommManaged.IPubSub myBCPS;
// TODO: what is this?
// public static void setPubSub(BlueCommManaged.IPubSub ps)
// {
// lock(typeof(PublisherCore))
// {
// BlueEventLog2.instance().assert(myBCPS == null,
// "PublisherCore.setPubSub() should be called only once and before any use of PublisherCore");
// myBCPS = ps;
// }
// }
public static string pubSubBlueCommPrefix = "ps/"; public static string pubSubBlueCommPrefix = "ps/";
public static string pubSubBlueCommTransactionPrefix = "tgps/"; public static string pubSubBlueCommTransactionPrefix = "tgps/";
public static string pubSubBlueCommTransactionComponentPrefix = "tcps/"; public static string pubSubBlueCommTransactionComponentPrefix = "tcps/";
// public static BlueCommManaged.IPubSub getPubSub()
// {
// lock (typeof(PublisherCore)) {
// return myBCPS;
// }
// }
enum Destination enum Destination {
{
None, None,
BlueComm, BlueComm,
PubSub, PubSub,
Both Both
} }
//private IPublisherCore myBlueCommPublisher = null; private IPublisherCore _publisher_core = null;
private IPublisherCore myPubSubPublisher = null;
private Dictionary<string, Destination> myPubDestinations =
new Dictionary<string, Destination>();
public PublisherCore() public PublisherCore() {
{ _publisher_core = new StaticPublisherCore();
myPubSubPublisher = new StaticPublisherCore();
//myBlueCommPublisher = new BlueCommPublisherCore(getPubSub());
} }
public PublisherCore(IPubSubRouter theRouter) public PublisherCore(IPubSubRouter theRouter) {
{ _publisher_core = new StaticPublisherCore(theRouter);
myPubSubPublisher = new StaticPublisherCore(theRouter);
//myBlueCommPublisher = new BlueCommPublisherCore(getPubSub());
} }
public PublisherCore(string URL) public PublisherCore(string URL) {
{ _publisher_core = new StaticPublisherCore(URL);
myPubSubPublisher = new StaticPublisherCore(URL);
//myBlueCommPublisher = new BlueCommPublisherCore(getPubSub());
} }
#region IPublisherCore Methods #region IPublisherCore Methods
public void Dispose() public void Dispose() {
{ _publisher_core.Dispose();
myPubSubPublisher.Dispose();
//myBlueCommPublisher.Dispose();
} }
public int getNumberOfWaitingMessages() { return myPubSubPublisher.getNumberOfWaitingMessages(); } public int getNumberOfWaitingMessages() { return _publisher_core.getNumberOfWaitingMessages(); }
private Destination myGetPublishDestionation (string tag) public void publish(string tag, Stream content) {
{
return Destination.PubSub;
// lock (this)
// {
// if (PubSub3Config.instance.BlueComm_mustDie)
// return Destination.PubSub;
// else
// {
// Destination d;
// return Destination.PubSub;
// if (myPubDestinations.TryGetValue(tag, out d))
// return d;
// else
// {
// foreach (string regexp in PubSub3Config.instance.BlueComm_PSTransition.tagsToPublishInPubSub)
// if (Regex.IsMatch(tag, regexp))
// {
// d = Destination.PubSub;
// foreach (string regexp2 in PubSub3Config.instance.BlueComm_PSTransition.exceptionsToTagsToPublishInPubSub)
// if (Regex.IsMatch(tag, regexp2))
// {
// d = Destination.None;
// break;
// }
// break;
// }
// if (!tag.StartsWith("mon"))
// foreach (string regexp in PubSub3Config.instance.BlueComm_PSTransition.tagsToPublishInBlueComm)
// if (Regex.IsMatch(tag, regexp))
// {
// if (d == Destination.PubSub)
// d = Destination.Both;
// else
// d = Destination.BlueComm;
// foreach (string regexp2 in PubSub3Config.instance.BlueComm_PSTransition.exceptionsToTagsToPublishInBlueComm)
// if (Regex.IsMatch(tag, regexp2))
// {
// if (d == Destination.Both)
// d = Destination.PubSub;
// else if (d == Destination.BlueComm)
// d = Destination.None;
// break;
// }
// break;
// }
// myPubDestinations[tag] = d;
// return d;
// }
// }
// }
}
public void publish(string tag, Stream content)
{
publish(tag, "", content); publish(tag, "", content);
} }
public void publish(string tagType, string tag, Stream content) public void publish(string tagType, string tag, Stream content) {
{ bts.Log.Gen.info($"PublisherCore.publish tag: {tag} stream.Length: {content.Length} stream.Position: {content.Position}");
Destination d = myGetPublishDestionation(tag); _publisher_core.publish(tagType, tag, content);
if (d == Destination.PubSub || d == Destination.Both)
myPubSubPublisher.publish(tagType, tag, content);
if (d == Destination.BlueComm || d == Destination.Both){}
//myBlueCommPublisher.publish(tagType, tag, content);
} }
public void beginTransaction() public void beginTransaction() {
{ _publisher_core.beginTransaction();
myPubSubPublisher.beginTransaction();
//myBlueCommPublisher.beginTransaction();
} }
public void commitTransaction() public void commitTransaction() {
{ _publisher_core.commitTransaction();
myPubSubPublisher.commitTransaction();
//myBlueCommPublisher.commitTransaction();
} }
public void abortTransaction() public void abortTransaction() {
{ _publisher_core.abortTransaction();
myPubSubPublisher.abortTransaction();
//myBlueCommPublisher.abortTransaction();
} }
public void clearState(string tagToClear) public void clearState(string tagToClear) {
{ _publisher_core.clearState(tagToClear);
Destination d = myGetPublishDestionation(tagToClear);
if (d == Destination.PubSub || d == Destination.Both)
myPubSubPublisher.clearState(tagToClear);
if (d == Destination.BlueComm || d == Destination.Both){}
//myBlueCommPublisher.clearState(tagToClear);
} }
public bool inTransaction public bool inTransaction { get { return _publisher_core.inTransaction; } }
{ get { return myPubSubPublisher.inTransaction; } }
#endregion #endregion
@@ -3,27 +3,25 @@ using System.IO;
using System.Collections; using System.Collections;
using BCG.Utilities; using BCG.Utilities;
namespace BCG namespace BCG {
{
/// <summary> /// <summary>
/// Controls (ie creates dynamically) an IPubsubDeserializer based on the type name which is /// Controls (ie creates dynamically) an IPubsubDeserializer based on the type name which is
/// part of the published data. Since its also a IPubsubDeserializer, all subsequent calls /// part of the published data. Since its also a IPubsubDeserializer, all subsequent calls
/// are passed though to the interface /// are passed though to the interface
/// </summary> /// </summary>
public class PubsubDeserializationController : IPubsubDeserializer public class PubsubDeserializationController : IPubsubDeserializer {
{
private IPubsubDeserializer myDeserializer; private IPubsubDeserializer myDeserializer;
public PubsubDeserializationController() { } public PubsubDeserializationController() { }
public void update(Stream stream) public void update(Stream stream) {
{ bts.Log.Gen.info("PubsubDeserializationController.update stream: {stream.Position} / {stream.Length}");
byte[] ops; byte[] ops;
Array rows; Array rows;
deserialize(new Stream[] { stream }, out ops, out rows); deserialize(new Stream[] { stream }, out ops, out rows);
} }
public void update(byte[] bytes) public void update(byte[] bytes) {
{ bts.Log.Gen.info("PubsubDeserializationController.update bytes: {bytes.Length}");
byte[] ops; byte[] ops;
Array rows; Array rows;
deserialize(new Stream[] { new MemoryStream(bytes, 0, bytes.Length, false, true) }, out ops, out rows); deserialize(new Stream[] { new MemoryStream(bytes, 0, bytes.Length, false, true) }, out ops, out rows);
@@ -31,39 +29,36 @@ namespace BCG
#region IPubsubDeserializer Members #region IPubsubDeserializer Members
public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) {
{
if (myDeserializer != null) if (myDeserializer != null) {
{
myDeserializer.deserialize(streams, out theOps, out theRows); myDeserializer.deserialize(streams, out theOps, out theRows);
} bts.Log.Gen.info($"PubsubDeserializationController.deserialize myDeserializer: {myDeserializer} ops: {theOps.Length} theRows: {theRows.Length}");
else } else {
{
Stream totalStream = UtilityMethods.getAggregatedStream(streams); Stream totalStream = UtilityMethods.getAggregatedStream(streams);
long initialPos = totalStream.Position; long initialPos = totalStream.Position;
BinaryReader br = new BinaryReader(totalStream); BinaryReader br = new BinaryReader(totalStream);
byte opCode = br.ReadByte(); byte opCode = br.ReadByte();
if (opCode == StreamOpCode.create) bts.Log.Gen.info($"PubsubDeserializationController.deserialize opCode: {opCode}");
{ if (opCode == StreamOpCode.create) {
string deserName = br.ReadString(); string deserName = br.ReadString();
if (deserName == "BCG.PubsubUniversalDeserializer") // so we can play old replay files which would not include the assembly name // TODOds
deserName = "BCG.PubsubUniversalDeserializer, DataStreamGateway"; if (deserName == "BCG.PubsubUniversalDeserializer, DataStreamGateway") // so we can play old replay files which would not include the assembly name
myDeserializer = Activator.CreateInstance(Type.GetType(deserName)) deserName = "BCG.PubsubUniversalDeserializer, DataStreamCore";
as IPubsubDeserializer; myDeserializer = Activator.CreateInstance(Type.GetType(deserName)) as IPubsubDeserializer;
totalStream.Position = initialPos; totalStream.Position = initialPos;
myDeserializer.deserialize(new Stream[] { totalStream }, out theOps, out theRows); myDeserializer.deserialize(new Stream[] { totalStream }, out theOps, out theRows);
} bts.Log.Gen.info($"PubsubDeserializationController.deserialize myDeserializer: {myDeserializer} ops: {theOps.Length} theRows: {theRows.Length}");
else
{ } else {
// BlueEventLog.logEvent("Received a non create before a create. Bad op code was " + opCode.ToString(), BlueEventLevel.Warning); bts.Log.Gen.error($"Received a non create before a create. Bad op code was : {opCode} ");
theOps = new byte[] { }; theOps = new byte[] { };
theRows = new object[] { }; theRows = new object[] { };
} }
} }
} }
public byte[] getState() public byte[] getState() {
{
return myDeserializer == null ? new byte[] { } : myDeserializer.getState(); return myDeserializer == null ? new byte[] { } : myDeserializer.getState();
} }
@@ -9,17 +9,14 @@ using System.IO;
using System.Diagnostics; using System.Diagnostics;
using mbts; using mbts;
namespace BCG namespace BCG {
{
public class ManagedTCPMessageHub : IPubsubMessageHub, public class ManagedTCPMessageHub : IPubsubMessageHub,
ITCPConnectionHandler, ITCPConnectionHandler,
ITCPMessageHandler ITCPMessageHandler {
{
private int myPort; private int myPort;
private bool myIsDisposed = false; private bool myIsDisposed = false;
public int port public int port {
{
get { return myPort; } get { return myPort; }
//property is read only //property is read only
@@ -66,8 +63,7 @@ namespace BCG
private DateTime myLastHeartbeatDeadCollectionTime = DateTime.Now; private DateTime myLastHeartbeatDeadCollectionTime = DateTime.Now;
private DateTime myLastDisposedDeadCollectionTime = DateTime.Now; private DateTime myLastDisposedDeadCollectionTime = DateTime.Now;
public ManagedTCPMessageHub(int port, TimeSpan hbSendInterval, int missingHBsForGC, TimeSpan collectDisposedConnectionsPeriod) public ManagedTCPMessageHub(int port, TimeSpan hbSendInterval, int missingHBsForGC, TimeSpan collectDisposedConnectionsPeriod) {
{
myPort = port; myPort = port;
myHeartbeatSendInterval = hbSendInterval; myHeartbeatSendInterval = hbSendInterval;
mySubscriberDropThreshold = new TimeSpan(0, 0, 0, 0, Convert.ToInt32(hbSendInterval.TotalMilliseconds * missingHBsForGC)); mySubscriberDropThreshold = new TimeSpan(0, 0, 0, 0, Convert.ToInt32(hbSendInterval.TotalMilliseconds * missingHBsForGC));
@@ -81,24 +77,21 @@ namespace BCG
#region IPubsubMessageHub Members #region IPubsubMessageHub Members
public void registerDataCallback(DataDelegate del) public void registerDataCallback(DataDelegate del) {
{
// BlueEventLog.logAssert(!myActivated, "connection callbacks should be registered before the call to activate"); // BlueEventLog.logAssert(!myActivated, "connection callbacks should be registered before the call to activate");
myDataCb = del; myDataCb = del;
} }
public void registerConnectionCallback(ConnectionStatusChange del) public void registerConnectionCallback(ConnectionStatusChange del) {
{
//BlueEventLog.logAssert(!myActivated, "data callbacks should be registered before the call to activate"); //BlueEventLog.logAssert(!myActivated, "data callbacks should be registered before the call to activate");
myConnCb = del; myConnCb = del;
} }
public void activate() public void activate() {
{
if (PubSub3Config.instance.PS3_BindToRandomPort || myPort == 0) if (PubSub3Config.instance.PS3_BindToRandomPort || myPort == 0)
myPort = ServerToClientManagedTCPConnectionAcceptor.activateWithRandomPort(this, myHeartbeatSendInterval); myPort = ServerToClientManagedTCPConnectionAcceptor.activateWithRandomPort(this, myHeartbeatSendInterval);
else else
myPort = ServerToClientManagedTCPConnectionAcceptor.activate(this, myPort, myHeartbeatSendInterval); myPort = ServerToClientManagedTCPConnectionAcceptor.accept_connections(this, myPort, myHeartbeatSendInterval);
string msg = String.Format("TCP Component is ready, listening on port {0} for incoming connections", myPort); string msg = String.Format("TCP Component is ready, listening on port {0} for incoming connections", myPort);
Console.WriteLine(msg); Console.WriteLine(msg);
@@ -108,19 +101,16 @@ namespace BCG
doItAllThread.Start(); doItAllThread.Start();
} }
public List<Pair<string, byte[]>> getUpdates(TimeSpan timeout, int maxElements) public List<Pair<string, byte[]>> getUpdates(TimeSpan timeout, int maxElements) {
{
return myReceivedData.dequeueMultipleElements(timeout, maxElements); return myReceivedData.dequeueMultipleElements(timeout, maxElements);
} }
public bool send(string receiver, byte[] data) public bool send(string receiver, byte[] data) {
{
mySendData.enqueue(new Pair<string, byte[]>(receiver, data)); mySendData.enqueue(new Pair<string, byte[]>(receiver, data));
return true; return true;
} }
public bool send(string receiver, System.IO.Stream str) public bool send(string receiver, System.IO.Stream str) {
{
BinaryReader br = new BinaryReader(str); BinaryReader br = new BinaryReader(str);
return send(receiver, br.ReadBytes(Convert.ToInt32(str.Length - str.Position))); return send(receiver, br.ReadBytes(Convert.ToInt32(str.Length - str.Position)));
} }
@@ -133,17 +123,13 @@ namespace BCG
/// Reports back connected and disconnected connections /// Reports back connected and disconnected connections
/// via the ConnectionStatucChange callback /// via the ConnectionStatucChange callback
/// </summary> /// </summary>
public void doWork() public void doWork() {
{ if (myConnCb != null) {
if (myConnCb != null) foreach (string nc in myNewConnectionURLs.dequeueAll()) {
{
foreach (string nc in myNewConnectionURLs.dequeueAll())
{
//BlueEventLog.logEvent(String.Format("doWork : New client {0} connected, making connection callback", nc), PS3LogLevels.connection); //BlueEventLog.logEvent(String.Format("doWork : New client {0} connected, making connection callback", nc), PS3LogLevels.connection);
myConnCb(nc, true); myConnCb(nc, true);
} }
foreach (string dc in myDeadConnectionURLs.dequeueAll()) foreach (string dc in myDeadConnectionURLs.dequeueAll()) {
{
//BlueEventLog.logEvent(String.Format("doWork : Client {0} disconnected, making disconnection callback", dc), PS3LogLevels.disconnection); //BlueEventLog.logEvent(String.Format("doWork : Client {0} disconnected, making disconnection callback", dc), PS3LogLevels.disconnection);
myConnCb(dc, false); myConnCb(dc, false);
} }
@@ -160,8 +146,7 @@ namespace BCG
/// stored in the hashtable and used for later communication) /// stored in the hashtable and used for later communication)
/// </summary> /// </summary>
/// <param name="connection"></param> /// <param name="connection"></param>
public void onConnection(ITCPConnection connection) public void onConnection(ITCPConnection connection) {
{
myNewConnections.enqueue(connection); myNewConnections.enqueue(connection);
} }
@@ -169,48 +154,38 @@ namespace BCG
#region ITCPMessageHandler Members #region ITCPMessageHandler Members
public void onData(RemoteMessage message) public void onData(RemoteMessage message) {
{
Console.WriteLine("ManagedTCPMessgageHub.onData recv msg :" + message.remoteConnectionInfo.ToString()); Console.WriteLine("ManagedTCPMessgageHub.onData recv msg :" + message.remoteConnectionInfo.ToString());
string sender = new BlutoSubAddress(message.remoteConnectionInfo.ip, message.remoteConnectionInfo.port).ToString(); string sender = new BlutoSubAddress(message.remoteConnectionInfo.ip, message.remoteConnectionInfo.port).ToString();
if (message.messageType == RemoteMessageType.data) if (message.messageType == RemoteMessageType.Data) {
{
if (myDataCb != null) if (myDataCb != null)
myDataCb(sender, message.message); myDataCb(sender, message.message);
else else
myReceivedData.enqueue(new Pair<string, byte[]>(sender, message.message)); myReceivedData.enqueue(new Pair<string, byte[]>(sender, message.message));
} } else if (message.messageType == RemoteMessageType.Heartbeat)
else if (message.messageType == RemoteMessageType.heartbeat)
myReceivedHeartbeats.enqueue(sender); myReceivedHeartbeats.enqueue(sender);
else else {
{
//BlueEventLog.logEvent("Unknown message type from the client", BlueEventLevel.Error); //BlueEventLog.logEvent("Unknown message type from the client", BlueEventLevel.Error);
} }
} }
#endregion #endregion
class TimmingInfo class TimmingInfo {
{ public TimmingInfo() {
public TimmingInfo()
{
myTimer = new Stopwatch(); myTimer = new Stopwatch();
reset(); reset();
} }
public void start() public void start() {
{ if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0) {
if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0)
{
myTimer.Reset(); myTimer.Reset();
myTimer.Start(); myTimer.Start();
} }
} }
public void stop() public void stop() {
{ if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0) {
if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0)
{
myTimer.Stop(); myTimer.Stop();
myMin = Math.Min(myMin, myTimer.ElapsedMilliseconds); myMin = Math.Min(myMin, myTimer.ElapsedMilliseconds);
myMax = Math.Max(myMax, myTimer.ElapsedMilliseconds); myMax = Math.Max(myMax, myTimer.ElapsedMilliseconds);
@@ -219,16 +194,14 @@ namespace BCG
} }
} }
public void reset() public void reset() {
{
myMin = Double.MaxValue; myMin = Double.MaxValue;
myMax = Double.MinValue; myMax = Double.MinValue;
myTotal = 0; myTotal = 0;
myNMeasurements = 0; myNMeasurements = 0;
} }
public override string ToString() public override string ToString() {
{
if (myNMeasurements > 0) if (myNMeasurements > 0)
return String.Format("min={0} max={1} avg={2} nMeasurements={3}", myMin, myMax, myTotal / myNMeasurements, myNMeasurements); return String.Format("min={0} max={1} avg={2} nMeasurements={3}", myMin, myMax, myTotal / myNMeasurements, myNMeasurements);
else else
@@ -242,8 +215,7 @@ namespace BCG
private double myTotal; private double myTotal;
} }
private void myDoItAll() private void myDoItAll() {
{
Dictionary<string, TimmingInfo> times = new Dictionary<string, TimmingInfo>(); Dictionary<string, TimmingInfo> times = new Dictionary<string, TimmingInfo>();
times.Add("myHandleNewConnections", new TimmingInfo()); times.Add("myHandleNewConnections", new TimmingInfo());
times.Add("myCollectDisposedConnections", new TimmingInfo()); times.Add("myCollectDisposedConnections", new TimmingInfo());
@@ -252,12 +224,9 @@ namespace BCG
DateTime lastTimmingLog = DateTime.Now; DateTime lastTimmingLog = DateTime.Now;
while (true) while (true) {
{ try {
try while (true) {
{
while (true)
{
///new connections ///new connections
times["myHandleNewConnections"].start(); times["myHandleNewConnections"].start();
myHandleNewConnections(); myHandleNewConnections();
@@ -279,16 +248,12 @@ namespace BCG
Pair<string, byte[]> p = mySendData.dequeue(myReadSendQueueTimeout); Pair<string, byte[]> p = mySendData.dequeue(myReadSendQueueTimeout);
times["send"].start(); times["send"].start();
if (p != null) if (p != null) {
{
ITCPConnection conn; ITCPConnection conn;
myConnections.TryGetValue(p.first, out conn); myConnections.TryGetValue(p.first, out conn);
if (conn != null) if (conn != null) {
{
conn.send(p.second); conn.send(p.second);
} } else {
else
{
string warningMsg = String.Format("sending data to {0} failed, this client might have " + string warningMsg = String.Format("sending data to {0} failed, this client might have " +
"disconnected, in which case it will be garbage collected within 60 secs", p.first); "disconnected, in which case it will be garbage collected within 60 secs", p.first);
//BlueEventLog.logEvent(warningMsg, PS3LogLevels.disconnection); //BlueEventLog.logEvent(warningMsg, PS3LogLevels.disconnection);
@@ -297,25 +262,21 @@ namespace BCG
times["send"].stop(); times["send"].stop();
if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0 && if (PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs > 0 &&
(DateTime.Now - lastTimmingLog).TotalSeconds > PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs) (DateTime.Now - lastTimmingLog).TotalSeconds > PubSub3Config.instance.PS3_DoItAllSecondsBetweenLogs) {
{
lastTimmingLog = DateTime.Now; lastTimmingLog = DateTime.Now;
StringBuilder b = new StringBuilder(); StringBuilder b = new StringBuilder();
b.AppendFormat("myDoItAll stats:\n"); b.AppendFormat("myDoItAll stats:\n");
b.AppendFormat("queue size: {0}\n", mySendData.getCount()); b.AppendFormat("queue size: {0}\n", mySendData.getCount());
foreach (string timerName in times.Keys) foreach (string timerName in times.Keys) {
{
b.AppendFormat("{0}: {1}\n", timerName, times[timerName]); b.AppendFormat("{0}: {1}\n", timerName, times[timerName]);
times[timerName].reset(); times[timerName].reset();
} }
//BlueEventLog.logEvent(b.ToString(), PS3LogLevels.performance); //BlueEventLog.logEvent(b.ToString(), PS3LogLevels.performance);
} }
} }
} } catch (Exception ex) {
catch (Exception ex)
{
//BlueEventLog.logEvent(ex); //BlueEventLog.logEvent(ex);
if (ex is OutOfMemoryException) if (ex is OutOfMemoryException)
throw ex; throw ex;
@@ -323,19 +284,15 @@ namespace BCG
} }
} }
private void myCollectDisposedConnections() private void myCollectDisposedConnections() {
{ if (DateTime.Now - myLastDisposedDeadCollectionTime > myCollectDisposedConnectionsPeriod) {
if (DateTime.Now - myLastDisposedDeadCollectionTime > myCollectDisposedConnectionsPeriod)
{
myLastDisposedDeadCollectionTime = DateTime.Now; myLastDisposedDeadCollectionTime = DateTime.Now;
List<string> disposedURLs = new List<string>(); List<string> disposedURLs = new List<string>();
foreach (KeyValuePair<string, ITCPConnection> p in myConnections) foreach (KeyValuePair<string, ITCPConnection> p in myConnections) {
{
if (p.Value.isDisposed) if (p.Value.isDisposed)
disposedURLs.Add(p.Key); disposedURLs.Add(p.Key);
} }
foreach (string url in disposedURLs) foreach (string url in disposedURLs) {
{
string disposeMsg = String.Format("Collecting disposed connections : {0} is disposed by the connection itself, " + string disposeMsg = String.Format("Collecting disposed connections : {0} is disposed by the connection itself, " +
"connection count before deleting this = {1}", url, myConnections.Count); "connection count before deleting this = {1}", url, myConnections.Count);
Console.WriteLine(disposeMsg); Console.WriteLine(disposeMsg);
@@ -346,10 +303,8 @@ namespace BCG
} }
} }
private void myCollectMissingHeartbeatConnections() private void myCollectMissingHeartbeatConnections() {
{ if (DateTime.Now - myLastHeartbeatDeadCollectionTime > mySubscriberDropThreshold) {
if (DateTime.Now - myLastHeartbeatDeadCollectionTime > mySubscriberDropThreshold)
{
myLastHeartbeatDeadCollectionTime = DateTime.Now; myLastHeartbeatDeadCollectionTime = DateTime.Now;
///these are all the hbs received bw the last ///these are all the hbs received bw the last
///and current collectDeadConnection calls ///and current collectDeadConnection calls
@@ -365,10 +320,9 @@ namespace BCG
foreach (string clientURL in myConnections.Keys) foreach (string clientURL in myConnections.Keys)
if (!indexedHbs.ContainsKey(clientURL)) if (!indexedHbs.ContainsKey(clientURL))
deadClients.Add(clientURL); deadClients.Add(clientURL);
foreach (string deadClient in deadClients) foreach (string deadClient in deadClients) {
{
string disposeMsg = String.Format("Collecting missing HB connections : {0} is dead due to missing hbs, Disposing it now, " + string disposeMsg = String.Format("Collecting missing HB connections : {0} is dead due to missing hbs, Disposing it now, " +
"connection count before deleting this = {1}", deadClient, myConnections.Count); "connection count before deleting this = {1}", deadClient, myConnections.Count);
//BlueEventLog.logEvent(disposeMsg, PS3LogLevels.disconnection); //BlueEventLog.logEvent(disposeMsg, PS3LogLevels.disconnection);
Console.WriteLine(disposeMsg); Console.WriteLine(disposeMsg);
myConnections[deadClient].Dispose(); myConnections[deadClient].Dispose();
@@ -378,11 +332,9 @@ namespace BCG
} }
} }
private void myHandleNewConnections() private void myHandleNewConnections() {
{
IList<ITCPConnection> newConns = myNewConnections.dequeueAll(); IList<ITCPConnection> newConns = myNewConnections.dequeueAll();
foreach (ITCPConnection conn in newConns) foreach (ITCPConnection conn in newConns) {
{
string ip; string ip;
int port; int port;
conn.getPeerAddress(out ip, out port); conn.getPeerAddress(out ip, out port);
@@ -399,13 +351,11 @@ namespace BCG
} }
#region IDisposable Memebers #region IDisposable Memebers
public void Dispose() public void Dispose() {
{
myIsDisposed = true; myIsDisposed = true;
} }
public bool isDisposed public bool isDisposed {
{
get { return myIsDisposed; } get { return myIsDisposed; }
} }
#endregion #endregion
@@ -4,23 +4,20 @@ using System.Text;
using BCG.Utilities.General; using BCG.Utilities.General;
using System.IO; using System.IO;
namespace BCG namespace BCG {
{
/// <summary> /// <summary>
/// Periodicaly publishes a heartbeat, under the tag name System_Heartbeat. /// Periodicaly publishes a heartbeat, under the tag name System_Heartbeat.
/// A subscriber can subscribe to it, and find out of a PubSub is alive or /// A subscriber can subscribe to it, and find out of a PubSub is alive or
/// not. Infact more so, if the subsriptions are working or not. /// not. Infact more so, if the subsriptions are working or not.
/// </summary> /// </summary>
internal class SystemHeartbeatPublisher : IWorker internal class SystemHeartbeatPublisher : IWorker {
{
public static string tagType = "System_Heartbeat"; public static string tagType = "System_Heartbeat";
public static string tagName = "System_Heartbeat"; public static string tagName = "System_Heartbeat";
private static SystemHeartbeatPublisher mySelf; private static SystemHeartbeatPublisher mySelf;
private SystemHeartbeatPublisher () private SystemHeartbeatPublisher() {
{
///this is using a multithreaded MasterWorker since the ///this is using a multithreaded MasterWorker since the
///regular one is called within the MessageReactor, which ///regular one is called within the MessageReactor, which
///waits 2500 msecs before calling doWork. This 2500 msecs ///waits 2500 msecs before calling doWork. This 2500 msecs
@@ -29,16 +26,14 @@ namespace BCG
MasterWorker.getInstance(true).addWorker(this, new TimeSpan(0, 0, 0, 0, 500)); MasterWorker.getInstance(true).addWorker(this, new TimeSpan(0, 0, 0, 0, 500));
} }
public static void init() public static void init() {
{
if (mySelf == null) if (mySelf == null)
mySelf = new SystemHeartbeatPublisher(); mySelf = new SystemHeartbeatPublisher();
} }
#region IWorker Members #region IWorker Members
public void doWork() public void doWork() {
{
PubMetaData pub = new PubMetaData(tagType, tagName, 1, "local", ClientInfo.thisClient.processString); PubMetaData pub = new PubMetaData(tagType, tagName, 1, "local", ClientInfo.thisClient.processString);
MetaData md = new MetaData(pub); MetaData md = new MetaData(pub);
MemoryStream ms = new MemoryStream(); MemoryStream ms = new MemoryStream();
@@ -16,10 +16,8 @@ using System.Collections.Generic;
using mbts; using mbts;
using mbts.spark; using mbts.spark;
namespace BCG namespace BCG {
{ public class PubSubMessageReactor {
public class PubSubMessageReactor
{
private ServerMessageHub myMessageHub; private ServerMessageHub myMessageHub;
private delegate void MyMsgHandler(string sender, Stream msg, MetaData m); private delegate void MyMsgHandler(string sender, Stream msg, MetaData m);
private Dictionary<string, MyMsgHandler> myOperationToHandler = new Dictionary<string, MyMsgHandler>(); private Dictionary<string, MyMsgHandler> myOperationToHandler = new Dictionary<string, MyMsgHandler>();
@@ -39,32 +37,24 @@ namespace BCG
/* This delegate allows pubsub components to be notified when the identity of the pubsub is known*/ /* This delegate allows pubsub components to be notified when the identity of the pubsub is known*/
public delegate void IdentityFoundDelegate(string identity, bool isRoot); public delegate void IdentityFoundDelegate(string identity, bool isRoot);
public event IdentityFoundDelegate onIdentityFound; public event IdentityFoundDelegate onIdentityFound;
public void fireIdentityFound(string identity, bool isRoot) public void fireIdentityFound(string identity, bool isRoot) {
{
if (onIdentityFound != null) if (onIdentityFound != null)
onIdentityFound(identity, isRoot); onIdentityFound(identity, isRoot);
} }
private IPubSubGovernor myGovernor;
public Dictionary<string, bool> myTagCompressionState = new Dictionary<string, bool>(); public Dictionary<string, bool> myTagCompressionState = new Dictionary<string, bool>();
public PubSubMessageReactor() : this(false, false, string.Empty, string.Empty) { } public PubSubMessageReactor() : this(false, false, string.Empty, string.Empty) { }
public PubSubMessageReactor(string q) : this(false, false, string.Empty, q) { } public PubSubMessageReactor(string q) : this(false, false, string.Empty, q) { }
public PubSubMessageReactor(bool clearQueue, bool bootEvent, string oldIP, string psq) public PubSubMessageReactor(bool clearQueue, bool bootEvent, string oldIP, string psq) {
{
myOperationToHandler.Add(typeof(PubMetaData).Name, new MyMsgHandler(myHandlePub)); myOperationToHandler.Add(typeof(PubMetaData).Name, new MyMsgHandler(myHandlePub));
myOperationToHandler.Add(typeof(SubMetaData).Name, new MyMsgHandler(myHandleSub)); myOperationToHandler.Add(typeof(SubMetaData).Name, new MyMsgHandler(myHandleSub));
myOperationToHandler.Add(typeof(UnSubMetaData).Name, new MyMsgHandler(myHandleUnsub)); myOperationToHandler.Add(typeof(UnSubMetaData).Name, new MyMsgHandler(myHandleUnsub));
myOperationToHandler.Add(typeof(TransactionMetaData).Name, new MyMsgHandler(myHandleTransaction)); myOperationToHandler.Add(typeof(TransactionMetaData).Name, new MyMsgHandler(myHandleTransaction));
// TODO, governor needs to be removed since its subststaion retired code
// if (PubSub3Config.instance.PS3_Components.governor != null)
// myGovernor = PubSubComponentFactory.buildGovernor(this);
//BlueEventLog.logEvent("System Initialized", PS3LogLevels.info); bts.Log.Gen.info("PubSubMessageReactor.cons System Initialized");
myReceivingProcThread = new Thread(new ThreadStart(myReceivingThreadProc)); myReceivingProcThread = new Thread(new ThreadStart(myReceivingThreadProc));
myMessageHub = ServerMessageHub.instance; myMessageHub = ServerMessageHub.instance;
@@ -75,93 +65,49 @@ namespace BCG
#region message handlers #region message handlers
public PubSubState state public PubSubState state {
{
get { return myState; } get { return myState; }
//this variable is read only //this variable is read only
} }
private void myHandlePub(string sender, Stream msg, MetaData m) // all publishes to the pubsub end here
{ private void myHandlePub(string sender, Stream msg_str, MetaData meta_data) {
bts.Log.Gen.info($"PubSubMessageReactor.myHandlePub sender: {sender} metadata: {meta_data} str.pos: {msg_str.Position} str.len: {msg_str.Length}");
mySingleTagTimeSliceTimer.start(); mySingleTagTimeSliceTimer.start();
///the position in the stream is set to where the data starts with, ///the position in the stream is set to where the data starts with,
///since the meta data was already read in the myProcessMsg ///since the meta data was already read in the myProcessMsg
PubMetaData pub = new PubMetaData(); PubMetaData pub = new PubMetaData();
m.copyTo(pub); meta_data.copyTo(pub);
pub.connection = sender; pub.connection = sender;
Console.WriteLine("PubSubMessageReactor.myHandlePub sender: " + sender + " tag: " + pub.tag);
//check to see if this publish took a while to get here
pub.warnIfLatencyExceedsThreshold(); pub.warnIfLatencyExceedsThreshold();
var subs_for_tag = myState.getSubscriberConnectionsForTag(pub.tag);
//check the governor to see if this publish is allowed, if we have a governor bts.Log.Gen.info($"PubSubMessageReactor.myHandlePub {subs_for_tag.Count} subs for tag: {pub.tag}");
PubSubGovernorResponse response = PubSubGovernorResponse.Accepted;
if (myGovernor != null)
response = myGovernor.requestToPublish(pub, msg);
if (response != PubSubGovernorResponse.Accepted)
{
// BlueEventLog2.instance().logEvent(string.Concat("PubSub Governor ",
// myGovernor.GetType().Name,
// " returned response ",
// response.ToString(),
// " publish on tag",
// pub.tag,
// "was ignored."), BlueEventLog2Level.Warning);
return;
}
//report the fact that we have a publish on this tag, if it's a debug tag
if (PubSub3Config.instance.isTagDebugTag(pub.tag))
{
// BlueEventLog.logEvent(string.Concat(GetType().Name,
// " got publish on tag ",
// pub.tag,
// " from app ", pub.publishingApp,
// " over connection ",
// pub.connection, ".\n",
// pub.isCompressed ? "Data was compressed.\n" : "Data was not compresed.\n",
// "total size: ",
// pub.size),
// PS3LogLevels.debug);
}
///for all the subscribers, sends the messages over straightaway ///for all the subscribers, sends the messages over straightaway
foreach (string address in myState.getSubscriberConnectionsForTag(pub.tag)) foreach (string address in myState.getSubscriberConnectionsForTag(pub.tag)) {
{
///the position is adjusted to zero, so as to include ///the position is adjusted to zero, so as to include
///the meta data as part of the message ///the meta data as part of the message
msg.Position = 0; msg_str.Position = 0;
myMessageHub.send(address, msg); bts.Log.Gen.info($"PubSubMessageReactor.myHandlePub sending tag: {pub.tag} to {address} len: {msg_str.Length - msg_str.Position}");
myMessageHub.send(address, msg_str);
} }
//reset the position of the message so we can decompress it appropriately //reset the position of the message so we can decompress it appropriately
msg.Position = 0; msg_str.Position = 0;
msg.Position = m.size(); msg_str.Position = meta_data.size();
if (pub.isCompressed) if (pub.isCompressed) {
{ long mdLength = msg_str.Position;
long mdLength = msg.Position; long compressedSize = msg_str.Length - mdLength;
long compressedSize = msg.Length - mdLength; if (compressedSize == 0) {
if (compressedSize == 0)
{
string errorMsg = "Error: tag " + pub.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update."; string errorMsg = "Error: tag " + pub.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update.";
//BlueEventLog.logEvent(errorMsg, BlueEventLevel.Error); bts.Log.Gen.error(errorMsg);
return; return;
} }
msg = StreamDecompressor.decompressStream(msg); msg_str = StreamDecompressor.decompressStream(msg_str);
long decompressedSize = msg.Length - mdLength; long decompressedSize = msg_str.Length - mdLength;
if (decompressedSize != pub.decompressedSize || compressedSize != pub.size) {
if (decompressedSize != pub.decompressedSize || compressedSize != pub.size)
{
string compressionWarning = string.Format("Warning: Stream for tag {0} decompressed from {1} to {2} bytes, with {3} byes of metadata. Metadata expected {4} to {5}. Tag state not updated.", string compressionWarning = string.Format("Warning: Stream for tag {0} decompressed from {1} to {2} bytes, with {3} byes of metadata. Metadata expected {4} to {5}. Tag state not updated.",
pub.tag, pub.tag,
compressedSize, compressedSize,
@@ -169,43 +115,30 @@ namespace BCG
mdLength, mdLength,
pub.size, pub.size,
pub.decompressedSize); pub.decompressedSize);
bts.Log.Gen.warn(compressionWarning);
//BlueEventLog.logEvent(compressionWarning, BlueEventLevel.Error);
return; return;
} }
} }
//give the user a notification of the compression status of the tag has changed. //give the user a notification of the compression status of the tag has changed.
if ((!(myTagCompressionState.ContainsKey(pub.tag)) || if ((!(myTagCompressionState.ContainsKey(pub.tag)) ||
myTagCompressionState[pub.tag] != pub.isCompressed)) myTagCompressionState[pub.tag] != pub.isCompressed)) {
{
string compressionMessage; string compressionMessage;
myTagCompressionState[pub.tag] = pub.isCompressed; myTagCompressionState[pub.tag] = pub.isCompressed;
if (pub.isCompressed) if (pub.isCompressed) {
{
compressionMessage = "Tag " + pub.tag + " is compressed."; compressionMessage = "Tag " + pub.tag + " is compressed.";
//BlueEventLog.logEvent(compressionMessage, PS3LogLevels.compressionRatio); //BlueEventLog.logEvent(compressionMessage, PS3LogLevels.compressionRatio);
} } else {
else
{
compressionMessage = "Tag " + pub.tag + " is not compressed."; compressionMessage = "Tag " + pub.tag + " is not compressed.";
//BlueEventLog.logEvent(compressionMessage, PS3LogLevels.compressionRatio); //BlueEventLog.logEvent(compressionMessage, PS3LogLevels.compressionRatio);
} }
} }
///updating state ///updating state
msg.Position = 0; msg_str.Position = 0;
msg.Position += m.size(); msg_str.Position += meta_data.size();
myState.updatePubData(pub, msg); myState.updatePubData(pub, msg_str);
mySingleTagTimeSliceTimer.stop(); mySingleTagTimeSliceTimer.stop();
double thisTagProcessingTime = mySingleTagTimeSliceTimer.lastIntervalTime; double thisTagProcessingTime = mySingleTagTimeSliceTimer.lastIntervalTime;
if (onSingleTagPub != null) if (onSingleTagPub != null)
onSingleTagPub(pub.tag, thisTagProcessingTime); onSingleTagPub(pub.tag, thisTagProcessingTime);
@@ -213,8 +146,8 @@ namespace BCG
} }
private void myHandleSub(string sender, Stream msg, MetaData m) private void myHandleSub(string sender, Stream msg, MetaData m) {
{ bts.Log.Gen.info($"PubSubMessageReactor.myHandleSub sender: {sender} metadata: {m} str.pos: {msg.Position} str.len: {msg.Length}");
SubMetaData sub = new SubMetaData(); SubMetaData sub = new SubMetaData();
m.copyTo(sub); m.copyTo(sub);
sub.connection = sender; sub.connection = sender;
@@ -222,12 +155,10 @@ namespace BCG
///update the state ///update the state
myState.sub(sub); myState.sub(sub);
///to all these new subscriber s, send the existing data ///to all these new subscriber s, send the existing data
foreach (string tag in sub.tags) foreach (string tag in sub.tags) {
{
byte[] data = myState.getContent(tag); byte[] data = myState.getContent(tag);
//ie data has been published once //ie data has been published once
if (data != null) if (data != null) {
{
string tagType = myState.getTagType(tag); string tagType = myState.getTagType(tag);
///passing "" as the 4th arg, as PubMeta is used both by the ///passing "" as the 4th arg, as PubMeta is used both by the
///publisher to publish data to the server (where the publisher ///publisher to publish data to the server (where the publisher
@@ -237,17 +168,14 @@ namespace BCG
///needed as of now, so its left black to save a few bytes ///needed as of now, so its left black to save a few bytes
PubMetaData pub; PubMetaData pub;
MemoryStream realContent; MemoryStream realContent;
if (PubSub3Config.instance.isTagCompressed(tag)) if (PubSub3Config.instance.isTagCompressed(tag)) {
{
SharpZipCompressor Kompessor = new SharpZipCompressor(); SharpZipCompressor Kompessor = new SharpZipCompressor();
realContent = new MemoryStream(); realContent = new MemoryStream();
Kompessor.compress(new MemoryStream(data), realContent); Kompessor.compress(new MemoryStream(data), realContent);
pub = new PubMetaData(tagType, tag, (int)realContent.Length, (int)data.Length, "", ""); pub = new PubMetaData(tagType, tag, (int)realContent.Length, (int)data.Length, "", "");
} } else {
else
{
realContent = new MemoryStream(data); realContent = new MemoryStream(data);
pub = new PubMetaData(tagType, tag, data.Length, "", ""); pub = new PubMetaData(tagType, tag, data.Length, "", "");
} }
@@ -262,8 +190,7 @@ namespace BCG
} }
} }
private void myHandleUnsub(string sender, Stream msg, MetaData m) private void myHandleUnsub(string sender, Stream msg, MetaData m) {
{
UnSubMetaData unsub = new UnSubMetaData(); UnSubMetaData unsub = new UnSubMetaData();
m.copyTo(unsub); m.copyTo(unsub);
if (unsub.connection.Trim().Length == 0) if (unsub.connection.Trim().Length == 0)
@@ -271,8 +198,7 @@ namespace BCG
myState.unsub(unsub); myState.unsub(unsub);
} }
private void myHandleTransaction(string sender, Stream msg, MetaData md) private void myHandleTransaction(string sender, Stream msg, MetaData md) {
{
mySingleTagTimeSliceTimer.start(); mySingleTagTimeSliceTimer.start();
TransactionMetaData t = new TransactionMetaData(); TransactionMetaData t = new TransactionMetaData();
md.copyTo(t); md.copyTo(t);
@@ -280,8 +206,7 @@ namespace BCG
///form the structure tag->byte[]* ///form the structure tag->byte[]*
Dictionary<string, byte[]> tagToMessages = new Dictionary<string, byte[]>(); Dictionary<string, byte[]> tagToMessages = new Dictionary<string, byte[]>();
BinaryReader br = new BinaryReader(msg); BinaryReader br = new BinaryReader(msg);
for (int i = 0; i < nMessages; i++) for (int i = 0; i < nMessages; i++) {
{
MetaData pubMd = new MetaData(msg); MetaData pubMd = new MetaData(msg);
PubMetaData pub = new PubMetaData(); PubMetaData pub = new PubMetaData();
pubMd.copyTo(pub); pubMd.copyTo(pub);
@@ -289,8 +214,7 @@ namespace BCG
byte[] tagData = new byte[pub.size]; byte[] tagData = new byte[pub.size];
msg.Read(tagData, 0, tagData.Length); msg.Read(tagData, 0, tagData.Length);
//report the fact that we have a publish on this tag, if it's a debug tag //report the fact that we have a publish on this tag, if it's a debug tag
if (PubSub3Config.instance.isTagDebugTag(pub.tag)) if (PubSub3Config.instance.isTagDebugTag(pub.tag)) {
{
// BlueEventLog.logEvent(string.Concat(GetType().Name, // BlueEventLog.logEvent(string.Concat(GetType().Name,
// " got publish on tag ", // " got publish on tag ",
// pub.tag, // pub.tag,
@@ -315,13 +239,11 @@ namespace BCG
} }
///this gives subscriber->tag* ///this gives subscriber->tag*
Dictionary<string, List<string>> subscriberToTags = myState.getSubscriberConnectionsForTags(tagToMessages.Keys); Dictionary<string, List<string>> subscriberToTags = myState.getSubscriberConnectionsForTags(tagToMessages.Keys);
foreach (KeyValuePair<string, List<string>> kvp in subscriberToTags) foreach (KeyValuePair<string, List<string>> kvp in subscriberToTags) {
{
MemoryStream subStream = new MemoryStream(); MemoryStream subStream = new MemoryStream();
TransactionMetaData subTransaction = new TransactionMetaData(kvp.Value.Count); TransactionMetaData subTransaction = new TransactionMetaData(kvp.Value.Count);
new MetaData(subTransaction).save(subStream); new MetaData(subTransaction).save(subStream);
foreach (string tag in kvp.Value) foreach (string tag in kvp.Value) {
{
byte[] tagData = tagToMessages[tag]; byte[] tagData = tagToMessages[tag];
PubMetaData pub = new PubMetaData(myState.getTagType(tag), tag, tagData.Length, "", ""); PubMetaData pub = new PubMetaData(myState.getTagType(tag), tag, tagData.Length, "", "");
new MetaData(pub).save(subStream); new MetaData(pub).save(subStream);
@@ -336,8 +258,7 @@ namespace BCG
double timeExpended = mySingleTagTimeSliceTimer.lastIntervalTime; double timeExpended = mySingleTagTimeSliceTimer.lastIntervalTime;
//Don't try to divide by zero //Don't try to divide by zero
int numTags = tagToMessages.Keys.Count; int numTags = tagToMessages.Keys.Count;
if (numTags != 0) if (numTags != 0) {
{
double timeExpendedPerTag = timeExpended / tagToMessages.Keys.Count; double timeExpendedPerTag = timeExpended / tagToMessages.Keys.Count;
if (onSingleTagPub != null) if (onSingleTagPub != null)
foreach (string tag in tagToMessages.Keys) foreach (string tag in tagToMessages.Keys)
@@ -351,18 +272,16 @@ namespace BCG
/// </summary> /// </summary>
/// <param name="msg"></param> /// <param name="msg"></param>
/// <returns></returns> /// <returns></returns>
private void myProcessMsg(string sender, Stream msg, out MetaData md) private void myProcessMsg(string sender, Stream msg, out MetaData md) {
{ md = new MetaData(msg); // msg's Posiiton moved to end of meta data
md = new MetaData(msg);
MyMsgHandler op = myOperationToHandler[md["op-code"]]; MyMsgHandler op = myOperationToHandler[md["op-code"]];
//BlueEventLog.logEvent(string.Format("process message {0}", m)); bts.Log.Gen.info($"PubSubMessageReactor.myProcessMsg sender: {sender} metadata: {md}");
op(sender, msg, md); op(sender, msg, md);
} }
#endregion #endregion
private void myHandleConnections(string client, bool connected) private void myHandleConnections(string client, bool connected) {
{
Console.WriteLine("PubSubMessageReactor.myHandleConnections client: " + client + " connected: " + connected); Console.WriteLine("PubSubMessageReactor.myHandleConnections client: " + client + " connected: " + connected);
///interested in disconnections only, as connections of no ///interested in disconnections only, as connections of no
///significance until they subscribe, in which case an entry ///significance until they subscribe, in which case an entry
@@ -371,13 +290,11 @@ namespace BCG
myState.removeConnection(client); myState.removeConnection(client);
} }
public void stop() public void stop() {
{
myStop = true; myStop = true;
} }
public void start() public void start() {
{
myReceivingProcThread.Start(); myReceivingProcThread.Start();
} }
@@ -385,8 +302,7 @@ namespace BCG
/// loops around and reads messages from the messageReceiver. calls processMsg /// loops around and reads messages from the messageReceiver. calls processMsg
/// passing in the IClientMessage. also logs stuff about the message. /// passing in the IClientMessage. also logs stuff about the message.
/// </summary> /// </summary>
private void myReceivingThreadProc() private void myReceivingThreadProc() {
{
TimeSpan receiveTimeout = new TimeSpan(0, 0, 0, 0, 2500); TimeSpan receiveTimeout = new TimeSpan(0, 0, 0, 0, 2500);
myMessageHub.activate(); myMessageHub.activate();
@@ -398,31 +314,25 @@ namespace BCG
string sender; string sender;
Stream msg; Stream msg;
MetaData meta; MetaData meta;
while (!myStop) while (!myStop) {
{
sender = null; sender = null;
msg = null; msg = null;
meta = null; meta = null;
try try {
{
List<Pair<string, byte[]>> receivedData = myMessageHub.getUpdates(receiveTimeout, 1); List<Pair<string, byte[]>> receivedData = myMessageHub.getUpdates(receiveTimeout, 1);
if (receivedData.Count > 0) if (receivedData.Count > 0) {
{
sender = receivedData[0].first; sender = receivedData[0].first;
msg = new MemoryStream(receivedData[0].second); msg = new MemoryStream(receivedData[0].second);
myTimer.start(); myTimer.start();
myProcessMsg(sender, msg, out meta); myProcessMsg(sender, msg, out meta);
myTimer.stop(); myTimer.stop();
if (myTimer.lastIntervalTime > myProcessingWarningThreshold) if (myTimer.lastIntervalTime > myProcessingWarningThreshold) {
{
string warningMsg = String.Format("processing metadata {0} took {1} msecs", meta, myTimer.lastIntervalTime); string warningMsg = String.Format("processing metadata {0} took {1} msecs", meta, myTimer.lastIntervalTime);
//BlueEventLog.logEvent(warningMsg, PS3LogLevels.performance); //BlueEventLog.logEvent(warningMsg, PS3LogLevels.performance);
} }
} }
myMasterWorker.doWork(); myMasterWorker.doWork();
} } catch (Exception e) {
catch (Exception e)
{
//BlueEventLog.logEvent(e, BlueEventLevel.Error); //BlueEventLog.logEvent(e, BlueEventLevel.Error);
Console.WriteLine("{0} : exception handling message: {1}\n", DateTime.Now, e.Message); Console.WriteLine("{0} : exception handling message: {1}\n", DateTime.Now, e.Message);
string errMsg = String.Format("Following information about the meta data and sender is available : " + string errMsg = String.Format("Following information about the meta data and sender is available : " +
@@ -7,21 +7,19 @@ using System.IO;
using mbts; using mbts;
namespace BCG namespace BCG {
{
/// <summary> /// <summary>
/// Mantains all the state that there is in the server. /// Mantains all the state that there is in the server.
/// This information is keyed off the tags (rather than /// This information is keyed off the tags (rather than
/// keyed off subscribers or publishers) /// keyed off subscribers or publishers)
/// State information consists of 2 distints sets : /// State information consists of 2 distinct sets :
/// myTagToData consists of all the data published to /// myTagToData consists of all the data published to
/// this pubsub. /// this pubsub.
/// myTagToTagInfo consists of all the subscription and /// myTagToTagInfo consists of all the subscription and
/// publishing information /// publishing information
/// </summary> /// </summary>
/// ///
public class PubSubState public class PubSubState {
{
public delegate void PubSubStateSingleTagChangeDelegate(PubSubSingleTagStateEvent theEvent); public delegate void PubSubStateSingleTagChangeDelegate(PubSubSingleTagStateEvent theEvent);
public event PubSubStateSingleTagChangeDelegate onSingleTagChange; public event PubSubStateSingleTagChangeDelegate onSingleTagChange;
@@ -33,8 +31,7 @@ namespace BCG
private Dictionary<string, SingleTagData> myTagToData = new Dictionary<string, SingleTagData>(); private Dictionary<string, SingleTagData> myTagToData = new Dictionary<string, SingleTagData>();
private Dictionary<string, SingleTagPubSubMap> myTagToPubsubMap = new Dictionary<string, SingleTagPubSubMap>(); private Dictionary<string, SingleTagPubSubMap> myTagToPubsubMap = new Dictionary<string, SingleTagPubSubMap>();
public PubSubState() public PubSubState() {
{
SystemSubscriptionMapPublisher.init(this); SystemSubscriptionMapPublisher.init(this);
SystemStatsPublisher.init(this); SystemStatsPublisher.init(this);
SystemHeartbeatPublisher.init(); SystemHeartbeatPublisher.init();
@@ -49,13 +46,10 @@ namespace BCG
/// </summary> /// </summary>
/// <param name="sub"></param> /// <param name="sub"></param>
/// <param name="md"></param> /// <param name="md"></param>
public void sub(SubMetaData sub) public void sub(SubMetaData sub) {
{
///tag based key ds update ///tag based key ds update
foreach (string tag in sub.tags) foreach (string tag in sub.tags) {
{ if (!myTagToPubsubMap.ContainsKey(tag)) {
if (!myTagToPubsubMap.ContainsKey(tag))
{
myTagToPubsubMap.Add(tag, new SingleTagPubSubMap(tag)); myTagToPubsubMap.Add(tag, new SingleTagPubSubMap(tag));
} }
myTagToPubsubMap[tag].onSub(sub); myTagToPubsubMap[tag].onSub(sub);
@@ -67,20 +61,15 @@ namespace BCG
onTagMapChanged(sub.tags); onTagMapChanged(sub.tags);
} }
public void unsub(UnSubMetaData unsub) public void unsub(UnSubMetaData unsub) {
{ foreach (string tag in unsub.tags) {
foreach (string tag in unsub.tags) if (myTagToPubsubMap.ContainsKey(tag)) {
{
if (myTagToPubsubMap.ContainsKey(tag))
{
///maybe check here, that if there are no subscribers ///maybe check here, that if there are no subscribers
///and publishers for the tag then remove it. ///and publishers for the tag then remove it.
myTagToPubsubMap[tag].onUnsub(unsub); myTagToPubsubMap[tag].onUnsub(unsub);
if (onSingleTagChange != null) if (onSingleTagChange != null)
onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribingApp)); onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribingApp));
} } else {
else
{
string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" + string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" +
" the unsub call by app {1} is voided", tag, unsub.unsubscribingApp); " the unsub call by app {1} is voided", tag, unsub.unsubscribingApp);
//BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning); //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
@@ -90,11 +79,9 @@ namespace BCG
onTagMapChanged(unsub.tags); onTagMapChanged(unsub.tags);
} }
public void removeConnection(string connection) public void removeConnection(string connection) {
{
List<string> tagsEffected = new List<string>(); List<string> tagsEffected = new List<string>();
foreach (KeyValuePair<string, SingleTagPubSubMap> p in myTagToPubsubMap) foreach (KeyValuePair<string, SingleTagPubSubMap> p in myTagToPubsubMap) {
{
///maybe check here, that if there are no subscribers ///maybe check here, that if there are no subscribers
///and publishers for the tag then remove it. ///and publishers for the tag then remove it.
bool wasPublisher = false; bool wasPublisher = false;
@@ -103,8 +90,7 @@ namespace BCG
if (p.Value.onConnectionRemoveIfExists(connection, out wasPublisher, out wasSubscriber)) if (p.Value.onConnectionRemoveIfExists(connection, out wasPublisher, out wasSubscriber))
tagsEffected.Add(tag); tagsEffected.Add(tag);
if (onSingleTagChange != null) if (onSingleTagChange != null) {
{
//we don't know the app or key qualifier or anything for these connection. that is unfortunate. //we don't know the app or key qualifier or anything for these connection. that is unfortunate.
if (wasPublisher) if (wasPublisher)
onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.publisherDie, connection, "")); onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.publisherDie, connection, ""));
@@ -128,67 +114,60 @@ namespace BCG
/// </summary> /// </summary>
/// <param name="tag"></param> /// <param name="tag"></param>
/// <param name="msg"></param> /// <param name="msg"></param>
public void updatePubData(PubMetaData pub, Stream msg) public void updatePubData(PubMetaData pub, Stream msg) {
{
int sz = (int)(msg.Length - msg.Position); int sz = (int)(msg.Length - msg.Position);
bts.Log.Gen.info($"PubSubData.updatePubData pub: msg : {msg.Position} / {msg.Length} sz: {sz}");
///this is the only place where SingleTagData should be created ///this is the only place where SingleTagData should be created
if (!myTagToData.ContainsKey(pub.tag)) if (!myTagToData.ContainsKey(pub.tag)) {
{
///first time data is published for this tag ///first time data is published for this tag
myTagToData.Add(pub.tag, new SingleTagData(pub.tag, pub.type)); myTagToData.Add(pub.tag, new SingleTagData(pub.tag, pub.type));
} }
myTagToData[pub.tag].pubUpdate(msg); myTagToData[pub.tag].pubUpdate(msg);
if (!myTagToPubsubMap.ContainsKey(pub.tag)) if (!myTagToPubsubMap.ContainsKey(pub.tag)) {
{
myTagToPubsubMap.Add(pub.tag, new SingleTagPubSubMap(pub.tag)); myTagToPubsubMap.Add(pub.tag, new SingleTagPubSubMap(pub.tag));
} }
bool pubChanged = false; bool pubChanged = false;
myTagToPubsubMap[pub.tag].onPub(pub, out pubChanged); myTagToPubsubMap[pub.tag].onPub(pub, out pubChanged);
if (pubChanged) if (pubChanged) {
{
if (onTagMapChanged != null) if (onTagMapChanged != null)
onTagMapChanged(new string[] { pub.tag }); onTagMapChanged(new string[] { pub.tag });
if (onSingleTagChange != null) if (onSingleTagChange != null)
onSingleTagChange( onSingleTagChange(
new PubSubSingleTagStateEvent(pub.tag, new PubSubSingleTagStateEvent(pub.tag,
PubSubSingleTagStateEvent.EventType.publish, PubSubSingleTagStateEvent.EventType.publish,
pub.connection, pub.publishingApp)); pub.connection, pub.publishingApp));
} }
if (onPublishStats != null) if (onPublishStats != null)
onPublishStats(pub.tag, sz); onPublishStats(pub.tag, sz);
} }
#endregion #endregion
public ICollection<string> getSubscriberConnectionsForTag(string tag) public ICollection<string> getSubscriberConnectionsForTag(string tag) {
{
if (myTagToPubsubMap.ContainsKey(tag)) if (myTagToPubsubMap.ContainsKey(tag))
return myTagToPubsubMap[tag].subscriberConnections; return myTagToPubsubMap[tag].subscriberConnections;
else else
return new string[] { }; return new string[] { };
} }
public ICollection<string> getSubscribingAppsForTag(string tag) public ICollection<string> getSubscribingAppsForTag(string tag) {
{
if (myTagToPubsubMap.ContainsKey(tag)) if (myTagToPubsubMap.ContainsKey(tag))
return myTagToPubsubMap[tag].subscripingApps; return myTagToPubsubMap[tag].subscripingApps;
else else
return new string[] { }; return new string[] { };
} }
public byte[] getContent(string tag) public byte[] getContent(string tag) {
{
if (!myTagToData.ContainsKey(tag)) if (!myTagToData.ContainsKey(tag))
return null; return null;
else else
return myTagToData[tag].getCurrentContent(); return myTagToData[tag].getCurrentContent();
} }
public string getTagType(string tag) public string getTagType(string tag) {
{
if (!myTagToData.ContainsKey(tag)) if (!myTagToData.ContainsKey(tag))
return null; return null;
else else
@@ -207,29 +186,23 @@ namespace BCG
/// <param name="deadPublishers"></param> /// <param name="deadPublishers"></param>
public void getAllConnections(string tag, out Dictionary<string, string> subscribers, public void getAllConnections(string tag, out Dictionary<string, string> subscribers,
out Dictionary<string, string> activePublishers, out Dictionary<string, string> activePublishers,
out ICollection<Pair<string, string>> deadPublishers) out ICollection<Pair<string, string>> deadPublishers) {
{ if (myTagToPubsubMap.ContainsKey(tag)) {
if (myTagToPubsubMap.ContainsKey(tag))
{
SingleTagPubSubMap map = myTagToPubsubMap[tag]; SingleTagPubSubMap map = myTagToPubsubMap[tag];
subscribers = map.subscribers; subscribers = map.subscribers;
activePublishers = map.activePublishers; activePublishers = map.activePublishers;
deadPublishers = map.deadPublishers; deadPublishers = map.deadPublishers;
} } else {
else
{
subscribers = activePublishers = new Dictionary<string, string>(); subscribers = activePublishers = new Dictionary<string, string>();
deadPublishers = new List<Pair<string, string>>(); deadPublishers = new List<Pair<string, string>>();
} }
} }
public ICollection<string> allTags public ICollection<string> allTags {
{
get { return myTagToPubsubMap.Keys; } get { return myTagToPubsubMap.Keys; }
} }
public ICollection<string> tagsWithData public ICollection<string> tagsWithData {
{
get { return myTagToData.Keys; } get { return myTagToData.Keys; }
} }
@@ -239,15 +212,11 @@ namespace BCG
/// </summary> /// </summary>
/// <param name="tags"></param> /// <param name="tags"></param>
/// <returns></returns> /// <returns></returns>
public Dictionary<string, List<string>> getSubscriberConnectionsForTags(IEnumerable<string> tags) public Dictionary<string, List<string>> getSubscriberConnectionsForTags(IEnumerable<string> tags) {
{
Dictionary<string, List<string>> retVal = new Dictionary<string, List<string>>(); Dictionary<string, List<string>> retVal = new Dictionary<string, List<string>>();
foreach (string tag in tags) foreach (string tag in tags) {
{ if (myTagToPubsubMap.ContainsKey(tag)) {
if (myTagToPubsubMap.ContainsKey(tag)) foreach (string sub in myTagToPubsubMap[tag].subscriberConnections) {
{
foreach (string sub in myTagToPubsubMap[tag].subscriberConnections)
{
if (!retVal.ContainsKey(sub)) if (!retVal.ContainsKey(sub))
retVal.Add(sub, new List<string>()); retVal.Add(sub, new List<string>());
retVal[sub].Add(tag); retVal[sub].Add(tag);
@@ -7,21 +7,18 @@ using System.Collections.Generic;
using BCG.Utilities; using BCG.Utilities;
using mbts; using mbts;
namespace BCG namespace BCG {
{
/// <summary> /// <summary>
/// SingleTagData consists of (as the name implies) /// SingleTagData consists of (as the name implies)
/// the data published for a tag /// the data published for a tag
/// </summary> /// </summary>
internal class SingleTagData internal class SingleTagData {
{
private string myTagType = null; private string myTagType = null;
private string myTag; private string myTag;
private byte[] myLastPublishedMsg; private byte[] myLastPublishedMsg;
private IPubSubServerTagHandler myHandler; private IPubSubServerTagHandler myHandler;
public SingleTagData(string tag, string tagType) public SingleTagData(string tag, string tagType) {
{
myTag = tag; myTag = tag;
myTagType = tagType; myTagType = tagType;
if (myTagType == SystemSubscriptionMapPublisher.tagType) if (myTagType == SystemSubscriptionMapPublisher.tagType)
@@ -29,6 +26,7 @@ namespace BCG
else if (myTagType == SystemStatsPublisher.tagType) else if (myTagType == SystemStatsPublisher.tagType)
myHandler = SystemStatsPublisher.getInstance(); myHandler = SystemStatsPublisher.getInstance();
else else
// this is where it gets the stream handler
myHandler = TagType2HandlerMap.getHandler(myTagType, true); myHandler = TagType2HandlerMap.getHandler(myTagType, true);
if (myHandler != null) if (myHandler != null)
myHandler.register(new InternalPublishDelegate(myHandleHandlerCallbacks)); myHandler.register(new InternalPublishDelegate(myHandleHandlerCallbacks));
@@ -38,16 +36,13 @@ namespace BCG
public string tagType { get { return myTagType; } } public string tagType { get { return myTagType; } }
internal void pubUpdate(Stream stream) internal void pubUpdate(Stream stream) {
{ string handler_str = myHandler != null ? myHandler.GetType().ToString() : "NULL";
if (myHandler != null) bts.Log.Gen.info($"SingleTagData.pubUpdate tag: {myTag} handler: {handler_str} stream: {stream.Position} / {stream.Length}");
{ if (myHandler != null) {
try try {
{
myHandler.update(myTag, new Stream[] { stream }); myHandler.update(myTag, new Stream[] { stream });
} } catch (Exception e) {
catch (Exception e)
{
// BlueEventLog.logEvent(e); // BlueEventLog.logEvent(e);
string errMsg = "tag handler threw an exception " + e.Message + " for tag type = " + myTagType + string errMsg = "tag handler threw an exception " + e.Message + " for tag type = " + myTagType +
" tag = " + myTag + " length = " + stream.Length + "\n" + " tag = " + myTag + " length = " + stream.Length + "\n" +
@@ -58,30 +53,22 @@ namespace BCG
else else
throw new ApplicationException(errMsg, e); throw new ApplicationException(errMsg, e);
} }
} } else {
else
{
myLastPublishedMsg = new byte[stream.Length - stream.Position]; myLastPublishedMsg = new byte[stream.Length - stream.Position];
stream.Read(myLastPublishedMsg, 0, myLastPublishedMsg.Length); stream.Read(myLastPublishedMsg, 0, myLastPublishedMsg.Length);
} }
} }
public byte[] getCurrentContent() public byte[] getCurrentContent() {
{ if (myHandler != null) {
if (myHandler != null)
{
return myHandler.getContent(myTag); return myHandler.getContent(myTag);
} } else {
else
{
return myLastPublishedMsg; return myLastPublishedMsg;
} }
} }
private void myHandleHandlerCallbacks(ICollection<Triplet<string, string, byte[]>> cbItems) private void myHandleHandlerCallbacks(ICollection<Triplet<string, string, byte[]>> cbItems) {
{ foreach (Triplet<string, string, byte[]> p in cbItems) {
foreach (Triplet<string, string, byte[]> p in cbItems)
{
string tagName = p.first, tagType = p.second; string tagName = p.first, tagType = p.second;
byte[] data = p.third; byte[] data = p.third;
PubMetaData pub = new PubMetaData(tagType, tagName, data.Length, "local", ClientInfo.thisClient.processString); PubMetaData pub = new PubMetaData(tagType, tagName, data.Length, "local", ClientInfo.thisClient.processString);
@@ -7,8 +7,7 @@ using BCG.Utilities.General;
using mbts; using mbts;
using mbts.spark; using mbts.spark;
namespace BCG namespace BCG {
{
/// <summary> /// <summary>
/// Deals with the subscriptions (and getting data) from one /// Deals with the subscriptions (and getting data) from one
/// pubsub server. Once again, theres no sharing of conenctions /// pubsub server. Once again, theres no sharing of conenctions
@@ -17,15 +16,13 @@ namespace BCG
/// its concerned with (passed as a conctructor arg). /// its concerned with (passed as a conctructor arg).
/// </summary> /// </summary>
internal interface ISingleServerSubscriberCore : IDisposable internal interface ISingleServerSubscriberCore : IDisposable {
{
IList<PubSubUpdate> getUpdates(int timeout); IList<PubSubUpdate> getUpdates(int timeout);
event PubSubConnectionChange onPubSubConnectionChange; event PubSubConnectionChange onPubSubConnectionChange;
void subscribe(params string[] tags); void subscribe(params string[] tags);
void unsubscribe(params string[] tags); void unsubscribe(params string[] tags);
} }
internal class SingleServerSubscriberCore : IDisposable, ISingleServerSubscriberCore internal class SingleServerSubscriberCore : IDisposable, ISingleServerSubscriberCore {
{
private HighPerformanceTimer myTimer = new HighPerformanceTimer(); private HighPerformanceTimer myTimer = new HighPerformanceTimer();
private PubSubServerConnectionFactory myConnectionFactory; private PubSubServerConnectionFactory myConnectionFactory;
private IPubSubServerConnection myConnection; private IPubSubServerConnection myConnection;
@@ -33,15 +30,13 @@ namespace BCG
public event PubSubConnectionChange onPubSubConnectionChange; public event PubSubConnectionChange onPubSubConnectionChange;
public SingleServerSubscriberCore(string serverURL, PubSubServerConnectionFactory connectionFactory) public SingleServerSubscriberCore(string serverURL, PubSubServerConnectionFactory connectionFactory) {
{
myConnectionFactory = connectionFactory; myConnectionFactory = connectionFactory;
myConnection = myConnectionFactory.getConnection(serverURL); myConnection = myConnectionFactory.getConnection(serverURL);
myConnection.registerCallback(new ConnectionStatusChange(myHandleServerReboot)); myConnection.registerCallback(new ConnectionStatusChange(myHandleServerReboot));
} }
public IList<PubSubUpdate> getUpdates() public IList<PubSubUpdate> getUpdates() {
{
return getUpdates(1); return getUpdates(1);
} }
@@ -51,54 +46,41 @@ namespace BCG
/// </summary> /// </summary>
/// <param name="timeout"></param> /// <param name="timeout"></param>
/// <returns></returns> /// <returns></returns>
public IList<PubSubUpdate> getUpdates(int timeout) public IList<PubSubUpdate> getUpdates(int timeout) {
{
myConnection.doWork(); myConnection.doWork();
List<PubSubUpdate> retVal = new List<PubSubUpdate>(); List<PubSubUpdate> retVal = new List<PubSubUpdate>();
IList<byte[]> dataItems = myConnection.getUpdates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue); IList<byte[]> dataItems = myConnection.getUpdates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue);
foreach (byte[] data in dataItems) foreach (byte[] data in dataItems) {
{
processMessage(data, retVal); processMessage(data, retVal);
} }
return retVal; return retVal;
} }
public List<string> getSubscribedTags() public List<string> getSubscribedTags() {
{
return myTags; return myTags;
} }
public void subscribe(params string[] tags) public void subscribe(params string[] tags) {
{ foreach (string tag in tags) {
foreach (string tag in tags) if (myTags.Contains(tag)) {
{
if (myTags.Contains(tag))
{
string errMsg = String.Format("tag {0} was already subscribed from server {1}", string errMsg = String.Format("tag {0} was already subscribed from server {1}",
tag, myConnection.serverURL); tag, myConnection.serverURL);
// BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning); // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning);
} } else {
else
{
myTags.Add(tag); myTags.Add(tag);
} }
} }
mySubscribe(tags); mySubscribe(tags);
} }
public void unsubscribe(params string[] tags) public void unsubscribe(params string[] tags) {
{
List<string> unsubTags = new List<string>(); List<string> unsubTags = new List<string>();
foreach (string tag in tags) foreach (string tag in tags) {
{ if (myTags.Contains(tag)) {
if (myTags.Contains(tag))
{
myTags.Remove(tag); myTags.Remove(tag);
unsubTags.Add(tag); unsubTags.Add(tag);
} } else {
else
{
string errMsg = String.Format("tag {0} is being unsubscribed from server {1}, eventhough " + string errMsg = String.Format("tag {0} is being unsubscribed from server {1}, eventhough " +
"it was never subscribed", tag, myConnection.serverURL); "it was never subscribed", tag, myConnection.serverURL);
// BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning); // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning);
@@ -108,13 +90,13 @@ namespace BCG
Stream str = new MemoryStream(); Stream str = new MemoryStream();
new MetaData(unsub).save(str); new MetaData(unsub).save(str);
str.Position = 0; str.Position = 0;
// this is where data is actually sent out
myConnection.send(str); myConnection.send(str);
} }
#region IDisposable Members #region IDisposable Members
public void Dispose() public void Dispose() {
{
///remove the subscriber ///remove the subscriber
//myConnection.send(new PubSubMessage(new MetaData(new RemoveSubscriberMetaData(myConnection.clientURL))).toStream()); //myConnection.send(new PubSubMessage(new MetaData(new RemoveSubscriberMetaData(myConnection.clientURL))).toStream());
myConnection.Dispose(); myConnection.Dispose();
@@ -126,8 +108,7 @@ namespace BCG
#endregion #endregion
private void mySubscribe(params string[] tags) private void mySubscribe(params string[] tags) {
{
SubMetaData sub = new SubMetaData(myConnection.clientURL, tags); SubMetaData sub = new SubMetaData(myConnection.clientURL, tags);
Stream str = new MemoryStream(); Stream str = new MemoryStream();
new MetaData(sub).save(str); new MetaData(sub).save(str);
@@ -135,20 +116,16 @@ namespace BCG
myConnection.send(str); myConnection.send(str);
} }
private void myHandleServerReboot(string serverURL, bool connection) private void myHandleServerReboot(string serverURL, bool connection) {
{ if (connection) {
if (connection)
{
//ie connection established again, so has to //ie connection established again, so has to
//resubscribe to all the tags //resubscribe to all the tags
// BlueEventLog.logEvent("Connection to " + serverURL + " established....resubscribing to all the tags", // BlueEventLog.logEvent("Connection to " + serverURL + " established....resubscribing to all the tags",
// PS3LogLevels.disconnection); // PS3LogLevels.disconnection);
mySubscribe(myTags.ToArray()); mySubscribe(myTags.ToArray());
} } else {
else
{
// BlueEventLog.logEvent("Connection to " + serverURL + " broke....attempting to reconnect...", // BlueEventLog.logEvent("Connection to " + serverURL + " broke....attempting to reconnect...",
// PS3LogLevels.disconnection); // PS3LogLevels.disconnection);
} }
@@ -158,18 +135,17 @@ namespace BCG
onPubSubConnectionChange(connection, serverURL); onPubSubConnectionChange(connection, serverURL);
} }
internal static void processMessage(byte[] data, List<PubSubUpdate> updates) internal static void processMessage(byte[] data, List<PubSubUpdate> updates) {
{ bts.Log.Gen.info($"SingleServerSubscriberCore.processMessage data_len: {data.Length}");
if (data != null && data.Length > 0) if (data != null && data.Length > 0) {
{
MemoryStream str = new MemoryStream(data); MemoryStream str = new MemoryStream(data);
MetaData md = new MetaData(str); MetaData md = new MetaData(str);
bts.Log.Gen.info($"SingleServerSubscriberCore.processMessage meta_data: {md}");
MemoryStream bodyStream = str; MemoryStream bodyStream = str;
long bodyPosition = str.Position; long bodyPosition = str.Position;
object op = Activator.CreateInstance (Type.GetType("BCG." + md["op-code"])); object op = Activator.CreateInstance(Type.GetType("BCG." + md["op-code"]));
md.copyTo(op); md.copyTo(op);
if (op is PubMetaData) if (op is PubMetaData) {
{
PubMetaData p = op as PubMetaData; PubMetaData p = op as PubMetaData;
//BCG.BlueTrace.WriteBlueCommReceiveW(p.tag, p.publishTimeTicks); //BCG.BlueTrace.WriteBlueCommReceiveW(p.tag, p.publishTimeTicks);
@@ -177,41 +153,29 @@ namespace BCG
p.warnIfLatencyExceedsThreshold(); p.warnIfLatencyExceedsThreshold();
//find out if we need to decompress this data //find out if we need to decompress this data
Stream realBody = bodyStream; Stream realBody = bodyStream;
if (p.isCompressed) if (p.isCompressed) {
{
long compressedSize = bodyStream.Length - bodyStream.Position; long compressedSize = bodyStream.Length - bodyStream.Position;
if (compressedSize == 0) if (compressedSize == 0) {
{
string errorMsg = "Error: tag " + p.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update."; string errorMsg = "Error: tag " + p.tag + " had compressed size of 0 bytes. This shouldnt' happen. Not decompressing tag or sending out update.";
Console.WriteLine(errorMsg); Console.WriteLine(errorMsg);
//BlueEventLog.logEvent(errorMsg, BlueEventLevel.Error); //BlueEventLog.logEvent(errorMsg, BlueEventLevel.Error);
return; return;
} }
realBody = StreamDecompressor.decompressStream(bodyStream); realBody = StreamDecompressor.decompressStream(bodyStream);
realBody.Position = bodyPosition; realBody.Position = bodyPosition;
} }
updates.Add(new PubSubUpdate(p.tag, p.type, realBody)); updates.Add(new PubSubUpdate(p.tag, p.type, realBody));
} } else if (op is TransactionMetaData) {
else if (op is TransactionMetaData)
{
TransactionMetaData tr = op as TransactionMetaData; TransactionMetaData tr = op as TransactionMetaData;
for (int i = 0; i < tr.nMessages; i++) for (int i = 0; i < tr.nMessages; i++) {
{
PubMetaData pub = new PubMetaData(); PubMetaData pub = new PubMetaData();
new MetaData(bodyStream).copyTo(pub); new MetaData(bodyStream).copyTo(pub);
byte[] trConstituent = new byte[pub.size]; byte[] trConstituent = new byte[pub.size];
bodyStream.Read(trConstituent, 0, trConstituent.Length); bodyStream.Read(trConstituent, 0, trConstituent.Length);
updates.Add(new PubSubUpdate(pub.tag, pub.type, new MemoryStream(trConstituent))); updates.Add(new PubSubUpdate(pub.tag, pub.type, new MemoryStream(trConstituent)));
} }
} } else { }
else{} //BlueEventLog.logAssert(false, "Unknown message received: " + md.ToString() + " , only Pub and TransactionMetaData are legal");
//BlueEventLog.logAssert(false, "Unknown message received: " + md.ToString() + " , only Pub and TransactionMetaData are legal");
} }
} }
} }
+146 -212
View File
@@ -15,91 +15,79 @@ using mbts;
* This file was originally called 'PublisherCore.cs'. It was renamed so that the publisher core could be replaced. * This file was originally called 'PublisherCore.cs'. It was renamed so that the publisher core could be replaced.
* *
* */ * */
namespace BCG namespace BCG {
{ // Provides a function to check if a tag is crushabel or not
public class CrushedTagManager public class CrushedTagManager {
{ private static CrushedTagManager _self;
private static CrushedTagManager mySelf; private Dictionary<string, bool> _tag_map = new Dictionary<string, bool>();
private Dictionary<string, bool> myTags = new Dictionary<string, bool>(); public static CrushedTagManager instance {
public static CrushedTagManager instance get {
{ if (_self == null)
get _self = new CrushedTagManager();
{ return _self;
if (mySelf == null)
mySelf = new CrushedTagManager();
return mySelf;
} }
} }
public bool isTagCrushable(string tag) public bool is_tag_crushable(string tag) {
{
tag = tag.ToLower(); tag = tag.ToLower();
lock (typeof(CrushedTagManager)) bool rv = false;
{ lock (typeof(CrushedTagManager)) {
if (!myTags.ContainsKey(tag)) if (!_tag_map.TryGetValue(tag, out rv)) {
{
foreach (string r in PubSub3Config.instance.PS3_CrushableTags) foreach (string r in PubSub3Config.instance.PS3_CrushableTags)
if (new System.Text.RegularExpressions.Regex(r).IsMatch(tag)) if (new System.Text.RegularExpressions.Regex(r).IsMatch(tag)) {
{ _tag_map.Add(tag, true);
myTags.Add(tag, true);
break; break;
} }
if (!myTags.ContainsKey(tag)) _tag_map.Add(tag, false);
myTags.Add(tag, false);
} }
} }
return myTags[tag]; return rv;
} }
} }
/// <summary> /// <summary>
/// All the data for a single tag, thats published within /// All the data for a single tag, thats published within
/// a transaction /// a transaction
/// </summary> /// </summary>
public class TagWaitingInTransaction public class TagWaitingInTransaction {
{ private bool _is_crushable;
private bool myIsCrushable; private List<byte[]> _buffered_data = new List<byte[]>();
private List<byte[]> myData = new List<byte[]>(); // this is the size of all the buffered data
private int myTotalSize = 0; private int _total_size = 0;
private string myTag, myTagType; private string _tag, _tag_type;
public TagWaitingInTransaction(string theTagType, string theTag) public TagWaitingInTransaction(string theTagType, string theTag) {
{ _tag = theTag;
myTag = theTag; _tag_type = theTagType;
myTagType = theTagType; _is_crushable = CrushedTagManager.instance.is_tag_crushable(_tag);
myIsCrushable = CrushedTagManager.instance.isTagCrushable(myTag);
} }
public void addPub(Stream pubStream) public void add_pub(Stream pub_stream) {
{ if (_is_crushable) {
if (myIsCrushable) // extract bytes, clear _buffered_data and queue to _buffered_data
{ byte[] new_data = new byte[(int)(pub_stream.Length - pub_stream.Position)];
byte[] newData = new byte[(int)(pubStream.Length - pubStream.Position)]; pub_stream.Read(new_data, 0, new_data.Length);
pubStream.Read(newData, 0, newData.Length); _buffered_data.Clear();
myData.Clear(); _buffered_data.Add(new_data);
myData.Add(newData); _total_size = new_data.Length;
myTotalSize = newData.Length; } else {
} // extract btes and queue to _buffered_data
else byte[] new_data = new byte[(int)(pub_stream.Length - pub_stream.Position)];
{ pub_stream.Read(new_data, 0, new_data.Length);
byte[] newData = new byte[(int)(pubStream.Length - pubStream.Position)]; _buffered_data.Add(new_data);
pubStream.Read(newData, 0, newData.Length); _total_size += new_data.Length;
myData.Add(newData);
myTotalSize += newData.Length;
} }
} }
public void writeTo(Stream s, string connection, string publishingApp) public void write_to(Stream s, string connection, string publishing_app) {
{ new MetaData(new PubMetaData(_tag_type, _tag, _total_size, connection, publishing_app)).save(s);
new MetaData(new PubMetaData(myTagType, myTag, myTotalSize, connection, publishingApp)).save(s); foreach (byte[] data in _buffered_data)
foreach (byte[] data in myData)
s.Write(data, 0, data.Length); s.Write(data, 0, data.Length);
} }
public string tag { get { return myTag; } } public string tag { get { return _tag; } }
public string tagType { get { return myTagType; } } public string tagType { get { return _tag_type; } }
public List<byte[]> data { get { return myData; } } public List<byte[]> data { get { return _buffered_data; } }
} }
public enum PublisherCoreMessageType public enum PublisherCoreMessageType {
{
transactionBegin, transactionBegin,
transactionCommit, transactionCommit,
transactionAbort, transactionAbort,
@@ -107,37 +95,35 @@ namespace BCG
clear clear
} }
public class PublisherCoreMessage public class PublisherCoreMessage {
{
public PublisherCoreMessageType msgType; public PublisherCoreMessageType msgType;
public string tagType, tag; public string tagType, tag;
public Stream data; public Stream data;
public PublisherCoreMessage(string tagToClear) public PublisherCoreMessage(string tagToClear) {
{
tag = tagToClear; tag = tagToClear;
msgType = PublisherCoreMessageType.clear; msgType = PublisherCoreMessageType.clear;
} }
public PublisherCoreMessage(PublisherCoreMessageType theMsgType) public PublisherCoreMessage(PublisherCoreMessageType theMsgType) {
{
msgType = theMsgType; msgType = theMsgType;
} }
public PublisherCoreMessage(string theTagType, string theTag, Stream theData) public PublisherCoreMessage(string theTagType, string theTag, Stream theData) {
{
msgType = PublisherCoreMessageType.publish; msgType = PublisherCoreMessageType.publish;
tagType = theTagType; tagType = theTagType;
tag = theTag; tag = theTag;
data = theData; data = theData;
} }
public override string ToString() {
return $"PublisherCoreMessage tag: {tag} data.Position: {data.Position} data.Length: {data.Length}";
}
} }
// this class uses an internal queue + thread to serialize and process calls // this class uses an internal queue + thread to serialize and process calls
// made through its public interface (IPublisherCore) // made through its public interface (IPublisherCore)
public class StaticPublisherCore : IPublisherCore public class StaticPublisherCore : IPublisherCore {
{
// used to allow for different transports // used to allow for different transports
// to be used when communicating to a pubsubserver // to be used when communicating to a pubsubserver
protected PubSubServerConnectionFactory myConnectionFactory = new PubSubServerConnectionFactory(false); protected PubSubServerConnectionFactory myConnectionFactory = new PubSubServerConnectionFactory(false);
@@ -172,57 +158,48 @@ namespace BCG
public StaticPublisherCore() public StaticPublisherCore()
// TODOds // TODOds
: this(new PubSubRouter()) : this(new PubSubRouter()) { }
{ }
public StaticPublisherCore(string serverURL) public StaticPublisherCore(string serverURL)
: this(new PubSubRouter(new PubSubRoute(serverURL, ".*"))) : this(new PubSubRouter(new PubSubRoute(serverURL, ".*"))) { }
{ }
public StaticPublisherCore(IPubSubRouter r) public StaticPublisherCore(IPubSubRouter r) {
{
myRouter = r; myRouter = r;
Thread pumpThread = new Thread(new ThreadStart(myMessagePump)); Thread pumpThread = new Thread(new ThreadStart(myMessagePump));
pumpThread.IsBackground = true; pumpThread.IsBackground = true;
pumpThread.Start(); pumpThread.Start();
} }
public void Dispose() public void Dispose() { }
{ }
#endregion #endregion
public int getNumberOfWaitingMessages() { return myMainQueue.getCount(); } public int getNumberOfWaitingMessages() { return myMainQueue.getCount(); }
#region Main Thread - all library functions are called on the main thread and queued to be picked by the pump #region Main Thread - all library functions are called on the main thread and queued to be picked by the pump
public void publish(string tag, Stream content) public void publish(string tag, Stream content) { publish("", tag, content); }
{ publish("", tag, content); }
public void publish(string tagType, string tag, Stream content) public void publish(string tagType, string tag, Stream content) {
{ PublisherCoreMessage msg = new PublisherCoreMessage(tagType, tag, content);
myMainQueue.enqueue(new PublisherCoreMessage(tagType, tag, content)); bts.Log.Gen.info($"StaticPublisherCore.publish msg: {msg}");
myMainQueue.enqueue(msg);
} }
public void beginTransaction() public void beginTransaction() {
{
myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionBegin)); myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionBegin));
} }
public void commitTransaction() public void commitTransaction() {
{
myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionCommit)); myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionCommit));
} }
public void abortTransaction() public void abortTransaction() {
{
myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionAbort)); myMainQueue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transactionAbort));
} }
public bool inTransaction public bool inTransaction { get { return myInTransaction; } }
{ get { return myInTransaction; } }
public void clearState(string tag) public void clearState(string tag) {
{
myMainQueue.enqueue(new PublisherCoreMessage(tag)); myMainQueue.enqueue(new PublisherCoreMessage(tag));
} }
@@ -230,70 +207,51 @@ namespace BCG
#region Message Pump Thread - this is where messages are dequeued and the action performed #region Message Pump Thread - this is where messages are dequeued and the action performed
private void myMessagePump() private void myMessagePump() {
{ while (true) {
while (true) try {
{ while (true) {
try
{
while (true)
{
object data = myMainQueue.dequeue(1000); object data = myMainQueue.dequeue(1000);
if (data != null) if (data != null) {
{
int qc = myMainQueue.getCount(); int qc = myMainQueue.getCount();
PublisherCoreMessage msg = data as PublisherCoreMessage; PublisherCoreMessage msg = data as PublisherCoreMessage;
if (msg.msgType == PublisherCoreMessageType.transactionBegin) bts.Log.Gen.info($"StaticPublisherCore.myMessagePump msg: {msg}");
{ if (msg.msgType == PublisherCoreMessageType.transactionBegin) {
myBeginTransaction(); myBeginTransaction();
} } else if (msg.msgType == PublisherCoreMessageType.transactionCommit) {
else if (msg.msgType == PublisherCoreMessageType.transactionCommit)
{
myCommitTransaction(); myCommitTransaction();
} } else if (msg.msgType == PublisherCoreMessageType.transactionAbort) {
else if (msg.msgType == PublisherCoreMessageType.transactionAbort)
{
myAbortTransaction(); myAbortTransaction();
} } else if (msg.msgType == PublisherCoreMessageType.publish) {
else if (msg.msgType == PublisherCoreMessageType.publish)
{
myPublish(msg.tagType, msg.tag, msg.data); myPublish(msg.tagType, msg.tag, msg.data);
} } else if (msg.msgType == PublisherCoreMessageType.clear) {
else if (msg.msgType == PublisherCoreMessageType.clear)
{
myClearState(msg.tag); myClearState(msg.tag);
} }
} }
foreach (IPubSubServerConnection conn in mySenders) foreach (IPubSubServerConnection conn in mySenders) {
{
conn.doWork(); conn.doWork();
} }
} }
} } catch (Exception ex) {
catch (Exception ex) bts.Log.Gen.error($"StaticPublisherCore.myMessagePump exception ex: {ex}");
{
// BlueEventLog.logEvent(ex);
if (ex is OutOfMemoryException) if (ex is OutOfMemoryException)
throw ex; throw ex;
} }
} }
} }
private void myBeginTransaction() private void myBeginTransaction() {
{
// BlueEventLog.logAssert(!inTransaction); // BlueEventLog.logAssert(!inTransaction);
//BlueEventLog.logEvent("StaticPublisherCore Starting Transaction.", BlueEventLevel.Information); //BlueEventLog.logEvent("StaticPublisherCore Starting Transaction.", BlueEventLevel.Information);
myInTransaction = true; myInTransaction = true;
} }
private void myCommitTransaction() private void myCommitTransaction() {
{
myEndTransaction(true); myEndTransaction(true);
} }
private void myAbortTransaction() private void myAbortTransaction() {
{
myEndTransaction(false); myEndTransaction(false);
} }
@@ -304,21 +262,17 @@ namespace BCG
//keep it consistent with the way it was done earlier), this should //keep it consistent with the way it was done earlier), this should
//simply publish it non transactionally (easy way out), or //simply publish it non transactionally (easy way out), or
//transactionally with respect to each pubsub //transactionally with respect to each pubsub
protected virtual void myEndTransaction(bool success) protected virtual void myEndTransaction(bool success) {
{ lock (this) {
lock (this)
{
// BlueEventLog.logAssert(inTransaction); // BlueEventLog.logAssert(inTransaction);
//BlueEventLog.logEvent("StaticPublisherCore Finishing Transaction.", BlueEventLevel.Information); //BlueEventLog.logEvent("StaticPublisherCore Finishing Transaction.", BlueEventLevel.Information);
myInTransaction = false; myInTransaction = false;
if (success) if (success) {
{
// pubsub server -> tags with updates // pubsub server -> tags with updates
Dictionary<string, List<string>> routesToTags = new Dictionary<string, List<string>>(); Dictionary<string, List<string>> routesToTags = new Dictionary<string, List<string>>();
// group updates per pubsub server // group updates per pubsub server
foreach (string tag in myWaitingInTransactionMessages.Keys) foreach (string tag in myWaitingInTransactionMessages.Keys) {
{
string thisURL = myRouter.getPubURLForTag(tag); string thisURL = myRouter.getPubURLForTag(tag);
if (!routesToTags.ContainsKey(thisURL)) if (!routesToTags.ContainsKey(thisURL))
routesToTags[thisURL] = new List<string>(); routesToTags[thisURL] = new List<string>();
@@ -327,17 +281,14 @@ namespace BCG
// more than one server means we won't be able to honor the transactionality request // more than one server means we won't be able to honor the transactionality request
// log a warning if that is the case. // log a warning if that is the case.
if (routesToTags.Keys.Count > 1) if (routesToTags.Keys.Count > 1) {
{
//this is going to be a long error message, but it's an importat one to be able to debug. //this is going to be a long error message, but it's an importat one to be able to debug.
StringBuilder errorMessageBuilder = new StringBuilder(); StringBuilder errorMessageBuilder = new StringBuilder();
errorMessageBuilder.Append("Tags within this transaction are routed to different pubSub servers.\n"); errorMessageBuilder.Append("Tags within this transaction are routed to different pubSub servers.\n");
errorMessageBuilder.Append("This will break the transactionality of clients receiving the data. Details:\n"); errorMessageBuilder.Append("This will break the transactionality of clients receiving the data. Details:\n");
foreach (string route in routesToTags.Keys) foreach (string route in routesToTags.Keys) {
{ foreach (string tag in routesToTags[route]) {
foreach (string tag in routesToTags[route])
{
errorMessageBuilder.Append("\tTag "); errorMessageBuilder.Append("\tTag ");
errorMessageBuilder.Append(tag); errorMessageBuilder.Append(tag);
errorMessageBuilder.Append(" is routed to "); errorMessageBuilder.Append(" is routed to ");
@@ -351,16 +302,14 @@ namespace BCG
} }
// iterate through each server with pending messages // iterate through each server with pending messages
foreach (KeyValuePair<string, List<string>> kvp in routesToTags) foreach (KeyValuePair<string, List<string>> kvp in routesToTags) {
{
IPubSubServerConnection transactionSender = null; IPubSubServerConnection transactionSender = null;
string publishingApp = ClientInfo.thisClient.processString; string publishingApp = ClientInfo.thisClient.processString;
string connection = kvp.Key; string connection = kvp.Key;
Stream transactionData = new MemoryStream(); Stream transactionData = new MemoryStream();
// iterate through each of the pending tag updates for one server // iterate through each of the pending tag updates for one server
foreach (string tag in kvp.Value) foreach (string tag in kvp.Value) {
{
///it would be the same transaction sender that would ///it would be the same transaction sender that would
///return each time ///return each time
transactionSender = myGetOrBuildSender(tag); transactionSender = myGetOrBuildSender(tag);
@@ -372,7 +321,7 @@ namespace BCG
// update internal state // update internal state
myState.publish(waiting.tag, waiting.tagType, data); myState.publish(waiting.tag, waiting.tagType, data);
// add a pubsub header and copy each message to the transactionData stream // add a pubsub header and copy each message to the transactionData stream
waiting.writeTo(transactionData, connection, publishingApp); waiting.write_to(transactionData, connection, publishingApp);
} }
// now transactionData holds all the updates for all the tags // now transactionData holds all the updates for all the tags
// that are to be sent to the PubSub represented by transactionSender // that are to be sent to the PubSub represented by transactionSender
@@ -384,36 +333,33 @@ namespace BCG
transactionData.Position = 0; transactionData.Position = 0;
// send the entire thing to the specific pubsub server. // send the entire thing to the specific pubsub server.
transactionSender.send(new PubSubMessage(new MetaData(tMd), transactionData).toStream()); // this is where the data is actually sent out !
var ps_msg = new PubSubMessage(new MetaData(tMd), transactionData);
Console.WriteLine($"serverConn.send metadata: {ps_msg.meta_data}");
bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
transactionSender.send(ps_msg);
} }
} }
myWaitingInTransactionMessages.Clear(); myWaitingInTransactionMessages.Clear();
} }
} }
protected virtual void myPublish(string tagType, string tag, Stream content) protected virtual void myPublish(string tagType, string tag, Stream content) {
{ bts.Log.Gen.info($"StaticPublisherCore.myPublish tag: {tag} content: {content.Position} / {content.Length} myInTransaction: {myInTransaction}");
if (myInTransaction) if (myInTransaction) {
{
///if its in transaction message is cached, and is sent ///if its in transaction message is cached, and is sent
///only when theres a commit or abort ///only when theres a commit or abort
if (!myWaitingInTransactionMessages.ContainsKey(tag)) if (!myWaitingInTransactionMessages.ContainsKey(tag))
myWaitingInTransactionMessages.Add(tag, new TagWaitingInTransaction(tagType, tag)); myWaitingInTransactionMessages.Add(tag, new TagWaitingInTransaction(tagType, tag));
myWaitingInTransactionMessages[tag].addPub(content); myWaitingInTransactionMessages[tag].add_pub(content);
} } else {
else
{
//if (CrushedTagManager.instance.isTagCrushable(tag))
{ {
myState.publish(tag, tagType, content); myState.publish(tag, tagType, content);
if (!myCrushableTagsWaitingOnAck.ContainsKey(tag)) if (!myCrushableTagsWaitingOnAck.ContainsKey(tag)) {
{ bts.Log.Gen.info("Sending crushable tag: {tag}");
// BlueEventLog.logEvent("Sending crushable tag " + tag, BlueEventLevel.Diagnosis);
mySendData(tag, tagType, content); mySendData(tag, tagType, content);
} } else {
else bts.Log.Gen.info("Could not send tag: {tag} since its waiting on an ack");
{
// BlueEventLog.logEvent("Could not send tag " + tag + " since its waiting on an ack", BlueEventLevel.Diagnosis);
myCrushableTagsWaitingOnAck[tag].second++; myCrushableTagsWaitingOnAck[tag].second++;
} }
} }
@@ -428,64 +374,60 @@ namespace BCG
} }
// add header, possibly compress the data, and sendto appropriate pubsub // add header, possibly compress the data, and sendto appropriate pubsub
private void mySendData(string tag, string tagType, Stream content) private void mySendData(string tag, string tagType, Stream content) {
{ bts.Log.Gen.info($"StaticPublisherCore.mySendData tag: {tag} tagType: {tagType}");
long initPos = content.Position; long initPos = content.Position;
// locate right pubsub for this tag // locate right pubsub for this tag
IPubSubServerConnection serverConn = myGetOrBuildSender(tag); IPubSubServerConnection serverConn = myGetOrBuildSender(tag);
Stream contentToPublish = content; Stream contentToPublish = content;
PubMetaData pubMd; PubMetaData pub_md;
if (PubSub3Config.instance.isTagCompressed(tag)) if (PubSub3Config.instance.isTagCompressed(tag)) {
{
contentToPublish = new MemoryStream(); contentToPublish = new MemoryStream();
SharpZipCompressor Kompressor = new SharpZipCompressor(); SharpZipCompressor Kompressor = new SharpZipCompressor();
Kompressor.compress(content, contentToPublish); Kompressor.compress(content, contentToPublish);
contentToPublish.Position = 0; contentToPublish.Position = 0;
pubMd = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString); pub_md = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
if (PubSub3Config.instance.shouldReportTagsCompressionRatio(tag)) if (PubSub3Config.instance.shouldReportTagsCompressionRatio(tag)) {
{
string msg = String.Format("Compressed tag {0} from {1} to {2} bytes, for a ratio of {3}.", tag, content.Length, contentToPublish.Length, (float)content.Length / (float)contentToPublish.Length); string msg = String.Format("Compressed tag {0} from {1} to {2} bytes, for a ratio of {3}.", tag, content.Length, contentToPublish.Length, (float)content.Length / (float)contentToPublish.Length);
// BlueEventLog.logEvent(msg, PS3LogLevels.compressionRatio); // BlueEventLog.logEvent(msg, PS3LogLevels.compressionRatio);
} }
} } else
else pub_md = new PubMetaData(tagType, tag, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
pubMd = new PubMetaData(tagType, tag, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString);
content.Position = initPos; content.Position = initPos;
if (CrushedTagManager.instance.isTagCrushable(tag)) if (CrushedTagManager.instance.is_tag_crushable(tag)) {
{
string refId = String.Empty; string refId = String.Empty;
Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.serverURL); Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.serverURL);
serverConn.sendWithConfirmation((new PubSubMessage(new MetaData(pubMd), contentToPublish).toStream() as MemoryStream).ToArray(), ref refId); // this is where the data is actually sent out (there are multiple sends)
PubSubMessage ps_msg = new PubSubMessage(new MetaData(pub_md), contentToPublish);
//Console.WriteLine($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}");
bts.Log.Gen.info($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}");
bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
serverConn.send_with_confirmation(ps_msg, ref refId);
myCrushableTagsWaitingOnAck.Add(tag, new Pair<string, int>(refId, 0)); myCrushableTagsWaitingOnAck.Add(tag, new Pair<string, int>(refId, 0));
} } else {
else //Console.WriteLine("StaticPublisherCore.mySend serverConn.send tag: " + tag + " server: " + serverConn.serverURL);
{ // this is where the data is actually sent out
Console.WriteLine("StaticPublisherCore.mySend serverConn.send tag: " + tag + " server: " + serverConn.serverURL); var ps_msg = new PubSubMessage(pub_md, contentToPublish);
serverConn.send(new PubSubMessage(new MetaData(pubMd), contentToPublish).toStream()); bts.Log.Gen.info($"StaticPublisherCore.send ps_msg: {ps_msg}");
serverConn.send(ps_msg);
} }
} }
private void myClearState(string tag) private void myClearState(string tag) {
{ if (myTagToSender.ContainsKey(tag)) {
if (myTagToSender.ContainsKey(tag))
{
myTagToSender.Remove(tag); myTagToSender.Remove(tag);
} }
myState.clear(tag); myState.clear(tag);
} }
protected virtual IPubSubServerConnection myGetOrBuildSender(string tag) protected virtual IPubSubServerConnection myGetOrBuildSender(string tag) {
{ if (!myTagToSender.ContainsKey(tag)) {
if (!myTagToSender.ContainsKey(tag))
{
string route = myRouter.getPubURLForTag(tag); string route = myRouter.getPubURLForTag(tag);
IPubSubServerConnection sender = myConnectionFactory.getConnection(route); IPubSubServerConnection sender = myConnectionFactory.getConnection(route);
myTagToSender.Add(tag, sender); myTagToSender.Add(tag, sender);
if (!mySenders.Contains(sender)) if (!mySenders.Contains(sender)) {
{
mySenders.Add(sender); mySenders.Add(sender);
sender.registerCallback(new ConnectionStatusChange(myHandleServerReboot)); sender.registerCallback(new ConnectionStatusChange(myHandleServerReboot));
sender.registerSendConfirmationCallback(new SendConfirmationDelegate(myOnSendConfirm)); sender.registerSendConfirmationCallback(new SendConfirmationDelegate(myOnSendConfirm));
@@ -497,15 +439,11 @@ namespace BCG
///in case of the server coming up, it publishes all the stored state ///in case of the server coming up, it publishes all the stored state
///for that server, back to it ///for that server, back to it
protected void myHandleServerReboot(string serverURL, bool connection) protected void myHandleServerReboot(string serverURL, bool connection) {
{ if (connection) {
if (connection)
{
string thisPublisherInfo = ClientInfo.thisClient.processString; string thisPublisherInfo = ClientInfo.thisClient.processString;
foreach (KeyValuePair<string, IPubSubServerConnection> p in myTagToSender) foreach (KeyValuePair<string, IPubSubServerConnection> p in myTagToSender) {
{ if (p.Value.serverURL == serverURL) {
if (p.Value.serverURL == serverURL)
{
string tagType; string tagType;
string tag = p.Key; string tag = p.Key;
myCrushableTagsWaitingOnAck.Clear(); myCrushableTagsWaitingOnAck.Clear();
@@ -547,16 +485,12 @@ namespace BCG
/// of waiting for ack /// of waiting for ack
/// </summary> /// </summary>
private Dictionary<string, Pair<string, int>> myCrushableTagsWaitingOnAck = new Dictionary<string, Pair<string, int>>(); private Dictionary<string, Pair<string, int>> myCrushableTagsWaitingOnAck = new Dictionary<string, Pair<string, int>>();
private void myOnSendConfirm(string messageId, bool confirmed) private void myOnSendConfirm(string messageId, bool confirmed) {
{ foreach (string tag in new List<string>(myCrushableTagsWaitingOnAck.Keys)) {
foreach (string tag in new List<string>(myCrushableTagsWaitingOnAck.Keys))
{
Pair<string, int> tagData = myCrushableTagsWaitingOnAck[tag]; Pair<string, int> tagData = myCrushableTagsWaitingOnAck[tag];
if (messageId == tagData.first) if (messageId == tagData.first) {
{
myCrushableTagsWaitingOnAck.Remove(tag); myCrushableTagsWaitingOnAck.Remove(tag);
if (tagData.second > 0) if (tagData.second > 0) {
{
// BlueEventLog.logEvent("Missed sending " + tagData.second + " messages, sending latest message now", BlueEventLevel.Diagnosis); // BlueEventLog.logEvent("Missed sending " + tagData.second + " messages, sending latest message now", BlueEventLevel.Diagnosis);
/// this data was waiting on an /// this data was waiting on an
/// ack, need to resend it /// ack, need to resend it
@@ -0,0 +1,132 @@
using System;
using System.Linq.Expressions;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading.Tasks;
using System.IO;
using Spark.Core.Reactive;
using System.Security.Cryptography.X509Certificates;
using BCG;
namespace BCG
{
// This uses AsyncTCPConnection2's stateful async functions
public class SimpleTCPClientAsync : ITCPConnectionHandler, ITCPMessageHandler
{
AsyncTCPConnection2 _conn = null;
int _m_msgs_received = 0;
public static async Task start(string ip, int port)
{
SimpleTCPClientAsync c = new SimpleTCPClientAsync(ip, port);
c.connect();
}
public void connect()
{
Task.Run(async () => await TCPSocketUtils.connect_to_server_async("127.0.0.1", 12345, this, TimeSpan.FromSeconds(2)));
}
public void onConnection(ITCPConnection connection)
{
Console.WriteLine("SimpleTCPClientAsync.onConnection connected to server, starting task to send data in a loop");
connection.register(this);
// start sending data
Task.Run(async () =>
{
byte[] buff = new byte[1024];
for (int i = 0; i < buff.Length; i++)
{
buff[i] = (byte)i;
}
_conn = connection as AsyncTCPConnection2;
while (!_conn.isDisposed)
{
await _conn.send_async(buff);
await Task.Delay(100);
}
});
}
public void onData(RemoteMessage msg)
{
_m_msgs_received++;
Console.WriteLine($"SimpleTCPClientAsync.onData n_msgs: {_m_msgs_received} msg: {msg}");
if (msg.messageType == RemoteMessageType.Disconnection)
{
connect();
}
}
public bool isDisposed { get { return false; } }
public void Dispose() { }
public SimpleTCPClientAsync(string ip, int port) { }
}
public class SimpleTCPClient : ITCPConnectionHandler, ITCPMessageHandler
{
private int _n_msgs_received = 0;
AsyncTCPConnection2 _conn = null;
public static void start(string ip, int port)
{
SimpleTCPClient c = new SimpleTCPClient(ip, port);
c.connect();
}
public void connect()
{
Console.WriteLine("SimpleTCPClient.onConnection connect() called");
ClientToServerManagedTCPConnector.connect_to(new BlutoSubAddress("bluto://127.0.0.1:12345"), this, TimeSpan.FromSeconds(1));
}
public void onConnection(ITCPConnection connection)
{
_conn = connection as AsyncTCPConnection2;
Console.WriteLine("SimpleTCPClient.onConnection connected to server, starting task to send data in a loop");
_conn.register(this);
byte[] buff = new byte[1024];
for (int i = 0; i < buff.Length; i++)
{
buff[i] = (byte)i;
}
while (!_conn.isDisposed)
{
_conn.send(buff);
System.Threading.Thread.Sleep(1000);
}
// Task.Run(async () =>
// {
// while (!_conn.isDisposed)
// {
// await _conn.send_async(buff);
// await Task.Delay(100);
// }
// });
}
public void onData(RemoteMessage msg)
{
_n_msgs_received++;
Console.WriteLine($"SimpleTCPClient.onData messages received: {_n_msgs_received} msg: {msg} messages received: {_n_msgs_received}");
if (msg.messageType == RemoteMessageType.Disconnection)
{
connect();
}
}
public bool isDisposed { get { return false; } }
public void Dispose() { }
public SimpleTCPClient(string ip, int port) { }
}
}
@@ -0,0 +1,123 @@
using BCG;
using Google.Protobuf;
using System;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using BCG;
using System.Xml.Serialization;
using System.Reflection;
using System.Data;
using System.Collections.Generic;
namespace BCG
{
public class SimpleTCPServerAsync : ITCPConnectionHandler, ITCPMessageHandler
{
int _n_msgs_received = 0;
SingleServerToClientManagedTCPConnectionAcceptor _acceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
IDictionary<RemoteConnectionInfo, ITCPConnection> _conn_map = new Dictionary<RemoteConnectionInfo, ITCPConnection>();
public static async Task start()
{
Console.WriteLine("SimpleTCPServer.start accepting socket connections");
SimpleTCPServerAsync s = new SimpleTCPServerAsync();
//await AsyncTCPConnection2.accept_connections("127.0.0.1", 12345, s, TimeSpan.FromMilliseconds(100));
s.accept();
}
// public static async Task init()
// {
// }
public void accept()
{
_acceptor.accept_on_port(this, "127.0.0.1", 12345, TimeSpan.Zero);
}
public void onConnection(ITCPConnection connection)
{
AsyncTCPConnection2 new_conn = connection as AsyncTCPConnection2;
RemoteConnectionInfo rci = new_conn.get_peer_address();
_conn_map.Add(rci, new_conn);
Console.WriteLine($"SimpleTCPServerAsync.onConnection new client connected conn: {rci}");
new_conn.register(this);
}
public void onData(RemoteMessage msg)
{
Console.WriteLine($"SimpleTCPServerAsync.onData msg: {msg}");
RemoteConnectionInfo rci = msg.remoteConnectionInfo;
ITCPConnection this_conn;
bool found = _conn_map.TryGetValue(rci, out this_conn);
if (!found)
{
Console.WriteLine("Could not find connection !!!!!!");
return;
}
_n_msgs_received++;
Console.WriteLine($"SimpleTCPServerAsync.onData msgs_received: {_n_msgs_received} msg: {msg} ci: {rci} echoing it back");
if (msg.messageType == RemoteMessageType.Data)
{
this_conn.send(msg.message);
// Task.Run(async () => {
// Console.WriteLine($"SimpleTCPServerAsync.onData sending msg: {msg} to client");
// await _conn.send_async(msg.message);
// });
}
}
public bool isDisposed { get { return false; } }
public void Dispose() { }
}
// public class SimpleTCPServer : ITCPConnectionHandler, ITCPMessageHandler {
// private AsyncTCPConnection2 _conn = null;
// private int _n_msgs_received = 0;
// SingleServerToClientManagedTCPConnectionAcceptor _acceptor = new SingleServerToClientManagedTCPConnectionAcceptor();
// public static async Task start() {
// Console.WriteLine("SimpleTCPServer.start accepting socket connections");
// SimpleTCPServer s = new SimpleTCPServer();
// s.accept();
// //await AsyncTCPConnection2.accept_connections("127.0.0.1", 12345, s, TimeSpan.FromMilliseconds(100));
// }
// public void accept() {
// _acceptor.accept_on_port(this, "127.0.0.1", 12345, TimeSpan.Zero);
// }
// public void onConnection(ITCPConnection connection) {
// Console.WriteLine("SimpleTCPServerAsync.onConnection new client connected");
// _conn = connection as AsyncTCPConnection2;
// connection.register(this);
// }
// public async void onData(RemoteMessage msg) {
// _n_msgs_received++;
// Console.WriteLine($"SimpleTCPServerAsync.onData msg: {msg} , msgs received: {_n_msgs_received} echoing it back");
// if (msg.messageType == RemoteMessageType.data && _conn != null) {
// Task.Run(async () => {
// //Console.WriteLine($"SimpleTCPServerAsync.onData sending msg: {msg} to client");
// //await _conn.send_async(msg.message);
// _conn.send(msg.message);
// });
// }
// }
// public bool isDisposed { get { return false; } }
// public void Dispose() { }
// }
}
@@ -0,0 +1,52 @@
using BCG;
using bts;
using System;
using System.IO;
using System.Runtime.Serialization;
namespace BCG {
public class TestPubItem : IPubItem {
int i = 100;
public static string test_tag = "TestTag";
public string getTag() {
return test_tag;
}
public void to_stream(Stream str) {
BinaryWriter bw = new BinaryWriter(str);
//bw.Write(d);
bw.Write(i);
}
}
public class TestPublisher {
TestPubItem _tbi = new TestPubItem();
IPublisherCore pcore = new StaticPublisherCore();
bool _hit_key_to_send = false;
public TestPublisher(bool hit_key_to_send) {
_hit_key_to_send = hit_key_to_send;
bts.Log.Gen.info($"Started TestPubliser _hit_key_to_send: {_hit_key_to_send}");
}
public void keep_publishing(TimeSpan wait_per, int n_pubs = System.Int32.MaxValue) {
TestPubItem pi = new TestPubItem();
for (int i = 0; i < n_pubs; i++) {
if (_hit_key_to_send) {
Console.WriteLine("Hit key to send data");
Console.ReadLine();
}
MemoryStream ms = new MemoryStream();
pi.to_stream(ms);
ms.Position = 0;
pcore.beginTransaction();
pcore.publish(TestPubItem.test_tag, ms);
pcore.commitTransaction();
if (!_hit_key_to_send)
System.Threading.Thread.Sleep(wait_per);
}
}
}
}
@@ -0,0 +1,31 @@
using BCG;
using bts;
using System;
using System.IO;
using System.Runtime.Serialization;
namespace BCG {
public class TestSubscriber {
SubscriberCore _sub_core = new SubscriberCore();
public TestSubscriber() {
bts.Log.Gen.info("Started TestSubscriber");
}
public void keep_polling(TimeSpan wait_per, int n_polls = System.Int32.MaxValue) {
_sub_core.subscribe(TestPubItem.test_tag);
for (int i = 0; i < n_polls; i++) {
var list = _sub_core.getUpdates();
if (list.Count > 0) {
Console.WriteLine($"Received {list.Count} items");
}
foreach (var p in list) {
Console.WriteLine($"Received item: {p} iter: {i}");
}
System.Threading.Thread.Sleep(wait_per);
}
}
}
}
+11 -12
View File
@@ -1,22 +1,24 @@
using System; using System;
using System.Threading; using System.Threading;
using System.Threading.Tasks;
using System.Linq; using System.Linq;
//using BCG.Utilities; //using BCG.Utilities;
//using BCG.Utilities.General; //using BCG.Utilities.General;
namespace BCG namespace BCG {
{ class PubSub2Main {
class PubSub2Main
{
/// <summary> /// <summary>
/// The main entry point for the application. /// The main entry point for the application.
/// </summary> /// </summary>
[STAThread] [STAThread]
public static void Main(string[] args) public static async Task Main(string[] args) {
{ bts.utils.BTSNativeMethods.provide_command_line(new string[] { }, 0);
mbts.config.Configuration.initialize();
// SimpleTCPServerAsync.start();
// Task.Delay(1000000000).Wait();
if (args.Length > 0 && args[0] == "daemon") if (args.Length > 0 && args[0] == "daemon")
args = args.Skip(1).ToArray(); args = args.Skip(1).ToArray();
@@ -26,14 +28,11 @@ namespace BCG
long currentMemory = 0; long currentMemory = 0;
int defaultWaitTime = 100; int defaultWaitTime = 100;
bool gced = false; bool gced = false;
new Thread(() => new Thread(() => {
{ while (true) {
while (true)
{
gced = false; gced = false;
currentMemory = GC.GetTotalMemory(false); currentMemory = GC.GetTotalMemory(false);
if (config.PS3_GCActivationThreshold > 0 && currentMemory > config.PS3_GCActivationThreshold) if (config.PS3_GCActivationThreshold > 0 && currentMemory > config.PS3_GCActivationThreshold) {
{
string gcMsg = String.Format("GCing memory, mem before GC = {0}", currentMemory); string gcMsg = String.Format("GCing memory, mem before GC = {0}", currentMemory);
// BlueEventLog.logEvent(gcMsg, PS3LogLevels.info); // BlueEventLog.logEvent(gcMsg, PS3LogLevels.info);
Console.WriteLine(gcMsg); Console.WriteLine(gcMsg);
@@ -1,19 +0,0 @@
syntax = "proto3";
package Bts.DataStream;
option csharp_namespace = "Bts.DataStream";
message DataStreamConfig {
repeated string assemblies = 1;
bool use_static_parser = 2;
bool log_blob_packets = 3;
bool is_test_mode = 4;
bool profile_streams = 5;
double profile_threshold = 6;
bool use_new_filter = 7;
bool use_new_map = 8;
bool use_increments_for_multiple_ops = 9;
bool use_uncopied_new_rows_in_fire = 10;
bool use_modularized_data_stream_bridge = 11;
}