diff --git a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs index 4ccda8101..bb2026f94 100644 --- a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs @@ -1,7 +1,12 @@ using System; - +using System.Collections.Generic; using MongoDB.Driver; using MongoDB.Bson; +using BCG; +using BCG.Utilities.General; +using ZstdSharp.Unsafe; +using System.Runtime.CompilerServices; +using mbts.spark; class Program { @@ -35,3 +40,103 @@ class Program { } } } + + +namespace BCG { + [DataStreamUserRespresentation("mongodb")] + public class MongoDBStream : ViewStream, IWorker { + private ViewStreamDatabase _vsd; + private readonly string _connection_string, _db_name, _collection_name; + private MongoClient _client; + private BlueTable _table; + + // Access the database + private IMongoDatabase _database; + private IDictionary _id_map = new Dictionary(); + // mongodb://192.168.1.80:27017 + + public MongoDBStream(string connection_string, string db_name, string collection_name) + : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) { + + } + public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) { + bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " + + $"db_name: {db_name}"); + _connection_string = connection_string; + _db_name = db_name; + _collection_name = collection_name; + _vsd = vsd; + _client = new MongoClient(_connection_string); + _database = _client.GetDatabase(_db_name); + } + + protected override void myOnFirstSubscriber() { + MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); + } + + protected override void myOnLastSubscriber() { + MasterWorker.getInstance(false).removeWorker(this); + } + + public void doWork() { + _read_mogodb(); + } + + private void _read_mogodb() { + HighPerformanceTimer timer = new HighPerformanceTimer(); + timer.start(); + // Access the collection + IMongoCollection collection = _database.GetCollection(name: _collection_name); + + // Fetch all documents from the collection + var documents = collection.Find(Builders.Filter.Empty).ToCursor(); + _table = _table == null ? null : new BlueTable(_table.schema); + // Iterate through the documents and print values + foreach (var document in documents.ToEnumerable()) { + bts.Log.Gen.info($"document: {document} json: {document.ToJson()}"); + if (_table == null) { + _init_table(document); + } + _add_row(document); + } + timer.stop(); + bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs"); + timer.start(); + var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true); + timer.stop(); + bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs"); + if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0) + return; + myFireEvent(dp); + } + + private void _init_table(BsonDocument doc_row) { + bts.Log.assert(_table == null); + List cis = new List(); + cis.Add((new ColumnInfo("id", 0, typeof(Int32)))); + foreach (var n in doc_row.Names) { + cis.Add(new ColumnInfo(n, cis.Count, typeof(string))); + } + IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name); + _table = new BlueTable(schema); + } + + private void _add_row(BsonDocument doc) { + var key_val = doc["_id"]; + var key = key_val.ToString(); + int id; + if (!_id_map.TryGetValue(key, out id)) { + id = _id_map.Count; + _id_map.Add(key, id); + } + MutableBlueRow nr = new MutableBlueRow(_table.schema); + nr.setValue("id", id); + foreach (var n in doc.Names) { + var val = doc[n].ToString(); + nr.setValue(n, val); + } + _table.addRow(nr); + } + } + +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs index f6ad09f6e..10f5b3ba2 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs @@ -14,56 +14,44 @@ using System.Linq.Expressions; using mbts; using BCG.Extensions; -namespace BCG -{ - public static class BlueDataUtility - { +namespace BCG { + public static class BlueDataUtility { #region Extensions - public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) - { + public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) { return BlueDataUtility.get_row_datetime_value(r, col_name, err_value); } - public static DateTime get_datetime(this IBlueRow r, int col_no, DateTime err_value = default(DateTime)) - { + public static DateTime get_datetime(this IBlueRow r, int col_no, DateTime err_value = default(DateTime)) { return BlueDataUtility.get_row_datetime_value(r, col_no, err_value); } - public static T get_enum(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct - { + public static T get_enum(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct { return BlueDataUtility.get_row_enum_value(r, col_name, err_value); } - public static T get_enum(this IBlueRow r, int col_no, T err_value = default(T)) where T : struct - { + public static T get_enum(this IBlueRow r, int col_no, T err_value = default(T)) where T : struct { return BlueDataUtility.get_row_enum_value(r, col_no, err_value); } - public static T get(this IBlueRow r, string col_name, T err_value = default(T)) - { + public static T get(this IBlueRow r, string col_name, T err_value = default(T)) { return get_row_value(r, col_name, err_value); } - public static T get(this IBlueRow r, int col_no, T err_value = default(T)) - { + public static T get(this IBlueRow r, int col_no, T err_value = default(T)) { return get_row_value(r, col_no, err_value); } - public static IEnumerable get_covalues(this IBlueRowCollection rows, string col) - { + public static IEnumerable get_covalues(this IBlueRowCollection rows, string col) { List rv = new List(); foreach (IBlueRow r in rows) rv.Add(r.get(col)); return rv; } - public static IEnumerable get_covalues(IEnumerable rows, string col) - { + public static IEnumerable get_covalues(IEnumerable rows, string col) { List rv = new List(); foreach (IBlueRow r in rows) rv.Add(r.get(col)); return rv; } - public static IBlueRowCollection where(this IBlueRowCollection rows, Func where_func) - { + public static IBlueRowCollection where(this IBlueRowCollection rows, Func where_func) { if (rows == null) return null; - else - { + else { BlueTable rv = new BlueTable(rows.schema); foreach (IBlueRow r in rows) if (where_func(r)) @@ -72,71 +60,61 @@ namespace BCG } } - public static IEnumerable to_ienum(this IBlueRowCollection rows) - { + public static IEnumerable to_ienum(this IBlueRowCollection rows) { List rv = new List(); foreach (IBlueRow r in rows) rv.Add(r); return rv; } - public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple[] cols) - { + public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple[] cols) { List cis = new List(input_schema.columnInfo); foreach (Tuple c in cols) cis.Add(new ColumnInfo(c.Item2, cis.Count, c.Item1)); return new BlueTableSchema(input_schema.tableName, cis.ToArray()); } - public static IBlueRowCollection to_collection(this IBlueRow r) - { + public static IBlueRowCollection to_collection(this IBlueRow r) { if (r == null) return null; - else - { + else { BlueTable rv = new BlueTable(r.schema); rv.addRow(r); return rv; } } - public static DataPacket build_dp(IBlueRowCollection new_rows, DataPacket last_fired_dp) - { + public static DataPacket build_dp(IBlueRowCollection new_rows, DataPacket last_fired_dp) { IBlueRowCollection added_rows = new_rows; IBlueRowCollection deleted_rows = last_fired_dp == null ? new BlueTable(new_rows.schema) : last_fired_dp.newRows; return new DataPacket(new_rows, added_rows, deleted_rows); } - public static DataPacket build_dp_inc(IBlueRowCollection new_rows, DataPacket last_fired_dp) - { - if (last_fired_dp == null) - { + public static DataPacket build_dp_inc(IBlueRowCollection new_rows, DataPacket last_fired_dp, bool deep_row_comapre = false) { + if (last_fired_dp == null) { IBlueRowCollection added_rows = new_rows; IBlueRowCollection deleted_rows = new BlueTable(new_rows.schema); return new DataPacket(new_rows, added_rows, deleted_rows); - } - else - { + } else { IBlueRowCollection added_rows = null; IBlueRowCollection deleted_rows = null; - compute_differences_id_based(new_rows, last_fired_dp.newRows, out deleted_rows, out added_rows); + if (deep_row_comapre) + compute_differences_fast(new_rows, last_fired_dp.newRows, out deleted_rows, out added_rows); + else + compute_differences_id_based(new_rows, last_fired_dp.newRows, out deleted_rows, out added_rows); return new DataPacket(new_rows, added_rows, deleted_rows); } } public static IBlueRowCollection adapt_for_sql_storage(this IBlueRowCollection rows, bool handle_datetime_min = false, - bool handle_double_nan = false) - { - if (rows != null) - { + bool handle_double_nan = false) { + if (rows != null) { BlueTable rv = new BlueTable(rows.schema); - foreach (IBlueRow r in rows) - { + foreach (IBlueRow r in rows) { MutableBlueRow mr = new MutableBlueRow(r); rv.addRow(mr); - foreach (ColumnInfo ci in rows.schema.columnInfo) - { + foreach (ColumnInfo ci in rows.schema.columnInfo) { if (handle_datetime_min && ci.colType == typeof(DateTime) && DateTime.MinValue.Equals(r[ci.colName])) mr.setValue(ci.colName, DBNull.Value); if (handle_double_nan && ci.colType == typeof(double) && Double.IsNaN(r.get(ci.colName))) @@ -144,25 +122,21 @@ namespace BCG } } return rv; - } - else + } else return null; } #endregion - public static void print_table(this IBlueRowCollection table, Func get_col_width) - { + public static void print_table(this IBlueRowCollection table, Func get_col_width) { var rv = get_printed_rows(table, get_col_width); Console.WriteLine(rv); } - public static string get_printed_rows(this IBlueRowCollection table, Func get_col_width) - { + public static string get_printed_rows(this IBlueRowCollection table, Func get_col_width) { // Print column headers string rv = String.Empty;// = $"{table.schema.tableName}______________________________________"; - foreach (var col_name in table.schema.columnNames) - { + foreach (var col_name in table.schema.columnNames) { int col_width = get_col_width(col_name); rv += $"{col_name.PadRight(col_width)} "; } @@ -170,15 +144,12 @@ namespace BCG //rv += "______________________________________"; // Print table data - foreach (IBlueRow row in table) - { - foreach (string col_name in table.schema.columnNames) - { + foreach (IBlueRow row in table) { + foreach (string col_name in table.schema.columnNames) { int col_width = get_col_width(col_name); object row_val = row[col_name]; string row_val_str = row_val.ToString(); - if (row_val != null) - { + if (row_val != null) { rv += $"{row_val_str.PadRight(col_width)} "; } } @@ -208,8 +179,7 @@ namespace BCG { TypeCode.UInt64, typeof(UInt64) } }; - public static IBlueTableSchema form_schema(string table_name, params Tuple[] cols) - { + public static IBlueTableSchema form_schema(string table_name, params Tuple[] cols) { List cis = new List(); foreach (Tuple c in cols) cis.Add(new ColumnInfo(c.Item1, cis.Count, _typecode_to_type[c.Item2])); @@ -225,17 +195,14 @@ namespace BCG /// /// /// - public static int getRowHashCode(IBlueRow row, bool isColAlphabetically) - { + public static int getRowHashCode(IBlueRow row, bool isColAlphabetically) { string retVal = ""; - if (isColAlphabetically) - { + if (isColAlphabetically) { string[] colNames = row.schema.columnNames; Array.Sort(colNames); foreach (string col in colNames) retVal += row[col].ToString(); - } - else + } else foreach (object o in row) retVal += o.ToString(); return retVal.GetHashCode(); @@ -247,8 +214,7 @@ namespace BCG /// /// /// - public static int getRowHashCode(IBlueRow row, params string[] colNames) - { + public static int getRowHashCode(IBlueRow row, params string[] colNames) { string retVal = ""; foreach (string colName in colNames) retVal += row[colName].ToString(); @@ -260,8 +226,7 @@ namespace BCG /// /// /// - public static DataTable toDataTable(IBlueRowCollection rows) - { + public static DataTable toDataTable(IBlueRowCollection rows) { DataTable retTable = getTableFromSchema(rows.schema); retTable.BeginLoadData(); foreach (IBlueRow row in rows) @@ -276,8 +241,7 @@ namespace BCG /// /// /// - public static DataRow toDataRow(IBlueRow row, DataTable table) - { + public static DataRow toDataRow(IBlueRow row, DataTable table) { DataRow retRow = table.NewRow(); foreach (string colName in row.schema.columnNames) retRow[colName] = row[colName]; @@ -300,8 +264,7 @@ namespace BCG /// table whos Schema is desired /// /// - public static IBlueTableSchema getSchema(DataTable table) - { + public static IBlueTableSchema getSchema(DataTable table) { ColumnInfo[] cis = new ColumnInfo[table.Columns.Count]; foreach (DataColumn col in table.Columns) cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType); @@ -314,8 +277,7 @@ namespace BCG /// /// /// - public static IBlueTableSchema getSchema(object o) - { + public static IBlueTableSchema getSchema(object o) { return getSchema(o.GetType()); } @@ -325,8 +287,7 @@ namespace BCG /// /// Base schema /// Columns to add - public static IBlueTableSchema addColumnsToSchema(IBlueTableSchema schema, params ColumnInfo[] cis) - { + public static IBlueTableSchema addColumnsToSchema(IBlueTableSchema schema, params ColumnInfo[] cis) { ArrayList rv = new ArrayList(schema.columnInfo); rv.AddRange(cis); return new BlueTableSchema(schema.tableName, rv.ToArray(typeof(ColumnInfo)) as ColumnInfo[]); @@ -338,8 +299,7 @@ namespace BCG /// /// /// - public static IBlueTableSchema appendSchemas(IBlueTableSchema s1, IBlueTableSchema appendedSchema) - { + public static IBlueTableSchema appendSchemas(IBlueTableSchema s1, IBlueTableSchema appendedSchema) { List cis = new List(s1.columnInfo); foreach (ColumnInfo c in appendedSchema.columnInfo) if (c.colName != "id") @@ -348,18 +308,15 @@ namespace BCG } - public static IBlueTableSchema getSchema(Type t) - { + public static IBlueTableSchema getSchema(Type t) { return getSchema(t, false); } - public static IBlueTableSchema getSchema(Type t, bool createIdField) - { + public static IBlueTableSchema getSchema(Type t, bool createIdField) { return getSchema(t, t.Name, createIdField); } - public static IBlueTableSchema getSchema(Type t, string tableName, bool createIdField) - { + public static IBlueTableSchema getSchema(Type t, string tableName, bool createIdField) { ArrayList colInfos = new ArrayList(); int i = 0; foreach (PropertyInfo pi in t.GetProperties()) @@ -379,20 +336,17 @@ namespace BCG tableName); } - public static IBlueTableSchema getSchema(Type t, string tableName) - { + public static IBlueTableSchema getSchema(Type t, string tableName) { return new BlueTableSchema(getSchema(t).columnInfo, tableName); } - public static IBlueRow createRowFromType(IBlueTableSchema schema, object o) - { + public static IBlueRow createRowFromType(IBlueTableSchema schema, object o) { return createRowFromType(schema, o, null); } //creates a row from the given type, and if idVal is not null, using the given id - public static IBlueRow createRowFromType(IBlueTableSchema schema, object o, object idVal) - { + public static IBlueRow createRowFromType(IBlueTableSchema schema, object o, object idVal) { object[] rowArray = new object[schema.columnInfo.Length]; Type t = o.GetType(); foreach (PropertyInfo pi in t.GetProperties()) @@ -411,8 +365,7 @@ namespace BCG } public static MutableBlueRow create_mutable_row(IBlueTableSchema schema, object o, object idVal, - Func type_converter) - { + Func type_converter) { object[] rowArray = new object[schema.columnInfo.Length]; Type t = o.GetType(); foreach (PropertyInfo pi in t.GetProperties()) @@ -431,8 +384,7 @@ namespace BCG } - public static IBlueTableSchema get_schema(Type t, string tableName, bool createIdField, Func type_converter) - { + public static IBlueTableSchema get_schema(Type t, string tableName, bool createIdField, Func type_converter) { List colInfos = new List(); int i = 0; if (createIdField) @@ -446,16 +398,13 @@ namespace BCG } public static MutableBlueRow create_mutable_row_not_slow(IBlueTableSchema schema, FieldInfo[] fields, PropertyInfo[] props, - object o, object idVal, Func data_converter) - { + object o, object idVal, Func data_converter) { object[] rowArray = new object[schema.columnInfo.Length]; - if (props != null) - { + if (props != null) { foreach (PropertyInfo pi in props) rowArray[schema[pi.Name].colNo] = data_converter(pi.PropertyType, pi.GetValue(o, null)); } - if (fields != null) - { + if (fields != null) { foreach (FieldInfo fi in fields) rowArray[schema[fi.Name].colNo] = data_converter(fi.FieldType, fi.GetValue(o)); } @@ -464,19 +413,16 @@ namespace BCG return new MutableBlueRow(schema, rowArray); } - public static T createTypeFromRow(IBlueRow row) - { + public static T createTypeFromRow(IBlueRow row) { return (T)createTypeFromRow(row, typeof(T)); } - public static object createTypeFromRow(IBlueRow row, Type t) - { + public static object createTypeFromRow(IBlueRow row, Type t) { object outValue = Activator.CreateInstance(t); PropertyInfo[] properties = t.GetProperties(); - foreach (PropertyInfo pi in properties) - { + foreach (PropertyInfo pi in properties) { if (myIsActiveProperty(pi)) if (row.Contains(pi.Name)) if (row[pi.Name].Equals(DBNull.Value) && !pi.PropertyType.Equals(typeof(Object))) @@ -485,8 +431,7 @@ namespace BCG pi.SetValue(outValue, row[pi.Name], null); } FieldInfo[] fields = t.GetFields(); - foreach (FieldInfo fi in fields) - { + foreach (FieldInfo fi in fields) { if (myIsActiveField(fi)) if (row.Contains(fi.Name)) if (row[fi.Name].Equals(DBNull.Value) && !fi.FieldType.Equals(typeof(Object))) @@ -501,18 +446,15 @@ namespace BCG //these functions should originally have been made public //but changing that would change a bunch of code - public static bool isActiveProperty(PropertyInfo pi) - { + public static bool isActiveProperty(PropertyInfo pi) { return myIsActiveProperty(pi); } - public static bool isActiveField(FieldInfo fi) - { + public static bool isActiveField(FieldInfo fi) { return myIsActiveField(fi); } - private static bool myIsActiveProperty(PropertyInfo pi) - { + private static bool myIsActiveProperty(PropertyInfo pi) { if (pi.GetGetMethod() == null) return false; foreach (Attribute att in pi.GetCustomAttributes(true)) @@ -521,8 +463,7 @@ namespace BCG return true; } - private static bool myIsActiveField(FieldInfo fi) - { + private static bool myIsActiveField(FieldInfo fi) { if (!fi.IsPublic) return false; foreach (Attribute att in fi.GetCustomAttributes(true)) @@ -531,8 +472,7 @@ namespace BCG return true; } - private static bool myIsActiveMember(MemberInfo mi) - { + private static bool myIsActiveMember(MemberInfo mi) { if (!(mi is FieldInfo) || !(mi is PropertyInfo) || (mi is PropertyInfo && (mi as PropertyInfo).GetGetMethod() == null) || @@ -544,21 +484,17 @@ namespace BCG return true; } - private static bool myDoesRowExistsInTable(IBlueRowCollection brc2, string[] columns, object[] oarray) - { + private static bool myDoesRowExistsInTable(IBlueRowCollection brc2, string[] columns, object[] oarray) { bool rowExists = false; - foreach (IBlueRow br in brc2) - { + foreach (IBlueRow br in brc2) { bool allElementsSame = true; - for (int i = 0; i < columns.Length; i++) - { + for (int i = 0; i < columns.Length; i++) { string col = columns[i]; object o1 = br[col]; object o2 = oarray[i]; allElementsSame = (o1.Equals(o2) == false) ? false : allElementsSame; } - if (allElementsSame == true) - { + if (allElementsSame == true) { rowExists = true; break; } @@ -573,8 +509,7 @@ namespace BCG /// Schema whose table needs to be created /// /// - public static DataTable getTableFromSchema(IBlueTableSchema schema) - { + public static DataTable getTableFromSchema(IBlueTableSchema schema) { DataTable retTable = new DataTable(schema.tableName); foreach (ColumnInfo ci in schema.columnInfo) retTable.Columns.Add(ci.colName, ci.colType); @@ -586,24 +521,20 @@ namespace BCG /// /// source row /// - public static IBlueRow toBlueRow(DataRow row) - { + public static IBlueRow toBlueRow(DataRow row) { return new BlueRow(getSchema(row), row); } - public static IBlueRowCollection toIBlueRowCollection(DataTable dt) - { + public static IBlueRowCollection toIBlueRowCollection(DataTable dt) { BlueTable retVal = new BlueTable(getSchema(dt)); foreach (DataRow dr in dt.Rows) retVal.addRow(dr); return retVal; } - public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) - { + public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) { BlueTable retVal = null; - foreach (DataRow dr in rows) - { + foreach (DataRow dr in rows) { IBlueRow r = toBlueRow(dr); if (retVal == null) retVal = new BlueTable(r.schema); @@ -620,8 +551,7 @@ namespace BCG /// /// /// - public static IBlueRow extractRow(IBlueRowCollection rows, int index) - { + public static IBlueRow extractRow(IBlueRowCollection rows, int index) { int i = 0; foreach (IBlueRow row in rows) if (i++ == index) @@ -635,15 +565,13 @@ namespace BCG /// /// /// - public static bool computeRowEquality(IBlueRow r1, IBlueRow r2) - { + public static bool computeRowEquality(IBlueRow r1, IBlueRow r2) { return computeRowEquality(r1, r2, r1.schema.columnNames); } public static bool computeRowEquality(IBlueRow r1, IBlueRow r2, - params string[] colNames) - { + params string[] colNames) { return compareRows(r1, r2, colNames) == 0 ? true : false; } @@ -654,8 +582,7 @@ namespace BCG /// /// /// - public static string rowToString(IBlueRow row, params string[] colNames) - { + public static string rowToString(IBlueRow row, params string[] colNames) { string retVal = ""; foreach (string colName in colNames) retVal += (row[colName] != null ? row[colName].ToString().Trim() : "NULL") + "\n"; @@ -664,12 +591,10 @@ namespace BCG public static string rowToString(IBlueRow row) { return rowToString(row, row.schema.columnNames); } - public static string rowCollectionToString(IBlueRowCollection rowColl) - { + public static string rowCollectionToString(IBlueRowCollection rowColl) { if (rowColl == null) return "Null IBlueRowCollection"; - else - { + else { string retVal = ""; foreach (IBlueRow r in rowColl) retVal += retVal == "" ? rowToString(r) : "\n" + rowToString(r); @@ -688,11 +613,9 @@ namespace BCG /// public static int compareRows(IBlueRow r1, IBlueRow r2, - params string[] colNames) - { + params string[] colNames) { int retVal = 0; - foreach (string colName in colNames) - { + foreach (string colName in colNames) { object val1 = r1[colName]; object val2 = r2[colName]; retVal = _compare_row_cols(val1, val2); @@ -702,16 +625,13 @@ namespace BCG return retVal; } - private static int _compare_row_cols(object v1, object v2) - { + private static int _compare_row_cols(object v1, object v2) { if (_is_null(v1)) return _is_null(v2) ? 0 : -1; - else - { + else { if (_is_null(v2)) return 1; - else - { + else { if (v1 is IComparable && v2 is IComparable) return (v1 as IComparable).CompareTo(v2); else if (v1 is byte[] && v2 is byte[]) @@ -722,8 +642,7 @@ namespace BCG } } - private static bool _compare_bytes(byte[] v1, byte[] v2) - { + private static bool _compare_bytes(byte[] v1, byte[] v2) { if (v1.Length != v2.Length) return false; else @@ -740,8 +659,7 @@ namespace BCG /// a row with the values to be converted /// the desired schema for the output row /// - public static IBlueRow changeSchema(IBlueRow inRow, IBlueTableSchema newSchema) - { + public static IBlueRow changeSchema(IBlueRow inRow, IBlueTableSchema newSchema) { object[] outputValues = new object[newSchema.columnInfo.Length]; for (int i = 0; i < outputValues.Length; ++i) @@ -756,8 +674,7 @@ namespace BCG /// /// /// - public static IBlueRowCollection combineTables(params IBlueRowCollection[] tables) - { + public static IBlueRowCollection combineTables(params IBlueRowCollection[] tables) { //get some base cases out of the way if (tables.Length == 0) return null; @@ -769,14 +686,12 @@ namespace BCG int numCols = tables[0].schema.columnInfo.Length; int idCol = tables[0].schema["id"].colNo; IBlueRowCollection previousTable = null; - foreach (IBlueRowCollection table in tables) - { + foreach (IBlueRowCollection table in tables) { if (previousTable != null) if (!compareSchemas(previousTable.schema, table.schema)) throw new ArgumentException("Error, tables must have the same schema to be combined"); - foreach (IBlueRow r in table) - { + foreach (IBlueRow r in table) { object[] values = new object[numCols]; r.CopyTo(values, 0); values[idCol] = idToUseNext++; @@ -796,8 +711,7 @@ namespace BCG /// /// /// - public static string[] compareRows(IBlueRow r1, IBlueRow r2) - { + public static string[] compareRows(IBlueRow r1, IBlueRow r2) { ArrayList retVal = new ArrayList(); foreach (string col in r1.schema.columnNames) if (!r1[col].Equals(r2[col])) @@ -805,8 +719,7 @@ namespace BCG return retVal.ToArray(typeof(string)) as string[]; } - public static BlueTable cloneRowCollection(IBlueRowCollection rows) - { + public static BlueTable cloneRowCollection(IBlueRowCollection rows) { BlueTable retVal = new BlueTable(rows.schema); foreach (IBlueRow r in rows) retVal.addRow(new BlueRow(r)); @@ -821,8 +734,7 @@ namespace BCG /// /// /// - public static IBlueRow getNewReplacedRow(IBlueRow r, object[] newValues, int[] indexes) - { + public static IBlueRow getNewReplacedRow(IBlueRow r, object[] newValues, int[] indexes) { if (!newValues.Length.Equals(indexes.Length)) throw new Exception("Debug Assert"); @@ -833,13 +745,11 @@ namespace BCG return new BlueRow(r.schema, rowArray); } - public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, int index) - { + public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, int index) { return getNewReplacedRow(r, new object[] { newValue }, new int[] { index }); } - public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, string colName) - { + public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, string colName) { return getNewReplacedRow(r, newValue, r.schema[colName].colNo); } @@ -849,8 +759,7 @@ namespace BCG /// /// /// - public static IBlueRow computeMaxRow(IBlueRowCollection rows, params string[] cols) - { + public static IBlueRow computeMaxRow(IBlueRowCollection rows, params string[] cols) { IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null; foreach (IBlueRow r in rows) retVal = compareRows(r, retVal, cols) > 0 ? r : retVal; @@ -863,8 +772,7 @@ namespace BCG /// /// /// - public static IBlueRow computeMinRow(IBlueRowCollection rows, params string[] cols) - { + public static IBlueRow computeMinRow(IBlueRowCollection rows, params string[] cols) { IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null; foreach (IBlueRow r in rows) retVal = compareRows(r, retVal, cols) < 0 ? r : retVal; @@ -877,24 +785,20 @@ namespace BCG /// /// /// - public static IBlueRow computeSum(IBlueRowCollection rows, params string[] cols) - { + public static IBlueRow computeSum(IBlueRowCollection rows, params string[] cols) { if (rows.Count == 0) return null; IBlueRow retVal = BlueDataUtility.extractRow(rows, 0); object[] sums = new object[cols.Length]; int[] idxs = new int[cols.Length]; - for (int i = 0; i < cols.Length; i++) - { + for (int i = 0; i < cols.Length; i++) { sums[i] = 0.0; idxs[i] = rows.schema[cols[i]].colNo; } - foreach (IBlueRow r in rows) - { - for (int i = 0; i < cols.Length; i++) - { + foreach (IBlueRow r in rows) { + for (int i = 0; i < cols.Length; i++) { double dval = BlueDataUtility.get_row_value(r, cols[i], 0.0); //r[cols[i]] == null || r[cols[i]] is DBNull ? 0.0 : (double)r[cols[i]]; sums[i] = (double)sums[i] + dval; @@ -905,8 +809,7 @@ namespace BCG } - public static IBlueRow computeSum(IBlueRowCollection rows, params string[] cols) - { + public static IBlueRow computeSum(IBlueRowCollection rows, params string[] cols) { if (rows.Count == 0) return null; IBlueRow retVal = BlueDataUtility.extractRow(rows, 0); if (rows.Count == 1) return retVal; @@ -914,16 +817,13 @@ namespace BCG object[] sums = new object[cols.Length]; int[] idxs = new int[cols.Length]; - for (int i = 0; i < cols.Length; i++) - { + for (int i = 0; i < cols.Length; i++) { sums[i] = default(T); idxs[i] = rows.schema[cols[i]].colNo; } - foreach (IBlueRow r in rows) - { - for (int i = 0; i < cols.Length; i++) - { + foreach (IBlueRow r in rows) { + for (int i = 0; i < cols.Length; i++) { T tval = r[cols[i]] == null || r[cols[i]] is DBNull ? default(T) : (T)r[cols[i]]; sums[i] = MagicAdder.Add((T)sums[i], tval); } @@ -938,16 +838,14 @@ namespace BCG /// /// /// - public static bool rowPresent(IBlueRowCollection table, IBlueRow row) - { + public static bool rowPresent(IBlueRowCollection table, IBlueRow row) { return table.containsID((int)row["id"]); } public static void computeDifferences(IBlueRowCollection newRows, IBlueRowCollection oldRows, out IBlueRowCollection dRows, - out IBlueRowCollection aRows) - { + out IBlueRowCollection aRows) { computeDifferences( newRows, oldRows, @@ -957,8 +855,7 @@ namespace BCG } public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows, - out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) - { + out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) { int[] col_indices = new int[new_rows.schema.columnNames.Length]; for (int i = 0; i < col_indices.Length; i++) col_indices[i] = i; @@ -966,17 +863,14 @@ namespace BCG } public static void compute_differences_id_based(IBlueRowCollection new_rows, IBlueRowCollection old_rows, - out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) - { + out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) { int[] col_indices = new int[] { new_rows.schema["id"].colNo }; compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows); } public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows, int[] col_indices, - out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) - { - Func rows_equal = (r1, r2) => - { + out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) { + Func rows_equal = (r1, r2) => { foreach (int col_no in col_indices) if (!(r1[col_no] ?? DBNull.Value).Equals(r2[col_no] ?? DBNull.Value)) return false; @@ -988,8 +882,7 @@ namespace BCG foreach (IBlueRow r in new_rows) if (!BlueDataUtility.rowPresent(old_rows, r)) add_rows.addRow(r); - else if (!rows_equal(r, old_rows[(int)r["id"]])) - { + else if (!rows_equal(r, old_rows[(int)r["id"]])) { add_rows.addRow(r); del_rows.addRow(old_rows[(int)r["id"]]); } @@ -1006,8 +899,7 @@ namespace BCG IBlueRowCollection c2, string[] columns, out IBlueRowCollection dRows, - out IBlueRowCollection aRows) - { + out IBlueRowCollection aRows) { dRows = new BlueTable(c1.schema); aRows = new BlueTable(c1.schema); @@ -1016,8 +908,7 @@ namespace BCG if (!BlueDataUtility.rowPresent(c2, nr)) (aRows as BlueTable).addRow(nr); else if (BlueDataUtility.compareRows(nr, c2[(int) - nr["id"]], columns) != 0) - { + nr["id"]], columns) != 0) { (aRows as BlueTable).addRow(nr); (dRows as BlueTable).addRow(c2[(int)nr["id"]]); @@ -1030,14 +921,11 @@ namespace BCG } //brcDiff = brc1 - brc2 - public static void diff(IBlueRowCollection brc1, IBlueRowCollection brc2, string[] columns, out IBlueRowCollection brcDiff) - { + public static void diff(IBlueRowCollection brc1, IBlueRowCollection brc2, string[] columns, out IBlueRowCollection brcDiff) { brcDiff = new BlueTable(brc1.schema); - foreach (IBlueRow br in brc1) - { + foreach (IBlueRow br in brc1) { object[] oarray = new object[columns.Length]; - for (int i = 0; i < columns.Length; i++) - { + for (int i = 0; i < columns.Length; i++) { string col = columns[i]; oarray[i] = br[col]; } @@ -1052,8 +940,7 @@ namespace BCG /// /// /// - public static void absorbChanges(BlueTable t, IBlueRowCollection dels, IBlueRowCollection adds) - { + public static void absorbChanges(BlueTable t, IBlueRowCollection dels, IBlueRowCollection adds) { foreach (IBlueRow r in dels) if (!t.deleteRow(r)) throw new Exception(String.Format("{0} does not have row with id {1} to delete", t.schema.tableName, r["id"])); @@ -1062,8 +949,7 @@ namespace BCG t.addRow(r); } - public static IBlueRowCollection toBlueRowCollection(IBlueRow r) - { + public static IBlueRowCollection toBlueRowCollection(IBlueRow r) { BlueTable rv = new BlueTable(r.schema); rv.addRow(r); return rv; @@ -1075,11 +961,9 @@ namespace BCG /// /// /// - public static IBlueRowCollection toBlueRowCollection(ICollection col) - { + public static IBlueRowCollection toBlueRowCollection(ICollection col) { BlueTable retVal = null; - foreach (IBlueRow r in col) - { + foreach (IBlueRow r in col) { retVal = retVal ?? new BlueTable(r.schema); retVal.addRow(r); } @@ -1098,23 +982,19 @@ namespace BCG IBlueRowCollection adds, out IBlueRowCollection truelyDels, out IBlueRowCollection truelyAdds, - string[] cols) - { + string[] cols) { truelyDels = new BlueTable(dels); truelyAdds = new BlueTable(adds); - foreach (IBlueRow r in dels) - { + foreach (IBlueRow r in dels) { if (adds.containsID((int)r["id"]) && - BlueDataUtility.computeRowEquality(adds[(int)r["id"]], r, cols)) - { + BlueDataUtility.computeRowEquality(adds[(int)r["id"]], r, cols)) { (truelyDels as BlueTable).deleteRow((int)r["id"]); (truelyAdds as BlueTable).deleteRow((int)r["id"]); } } } - public static IBlueRowCollection getSerializableCollection(IBlueRowCollection coll) - { + public static IBlueRowCollection getSerializableCollection(IBlueRowCollection coll) { BlueTable retVal = new OrderedBlueTable(coll.schema); foreach (IBlueRow r in coll) retVal.addRow(new BlueRow(r)); @@ -1127,8 +1007,7 @@ namespace BCG /// /// /// - public static ArrayList extractColumn(IBlueRowCollection rows, string colName) - { + public static ArrayList extractColumn(IBlueRowCollection rows, string colName) { ArrayList rv = new ArrayList(); foreach (IBlueRow r in rows) rv.Add(r[colName]); @@ -1147,19 +1026,16 @@ namespace BCG public static void checkConsistency(IBlueRowCollection oldState, IBlueRowCollection newRows, IBlueRowCollection delRows, - IBlueRowCollection addedRows) - { + IBlueRowCollection addedRows) { BlueTable calc = new BlueTable(oldState); Debug.Assert(oldState != null && newRows != null && delRows != null && addedRows != null); - foreach (IBlueRow r in delRows) - { + foreach (IBlueRow r in delRows) { Debug.Assert(calc.containsID((int)r["id"]), "deleted row's id = " + r["id"] + " does not exist"); calc.deleteRow(r); } - foreach (IBlueRow r in addedRows) - { + foreach (IBlueRow r in addedRows) { Debug.Assert(!calc.containsID((int)r["id"]), "added rows's id = " + r["id"].ToString() + " already exists"); calc.addRow(r); } @@ -1176,8 +1052,7 @@ namespace BCG /// /// /// - public static object[] getSelectedValues(IBlueRow r, params string[] colNames) - { + public static object[] getSelectedValues(IBlueRow r, params string[] colNames) { object[] retVal = new object[colNames.Length]; for (int i = 0; i < colNames.Length; i++) retVal[i] = r[colNames[i]]; @@ -1193,10 +1068,8 @@ namespace BCG /// /// /// - public static IBlueRow getRowWithKey(IBlueRowCollection rows, string[] keyColumns, object[] colValues) - { - foreach (IBlueRow r in rows) - { + public static IBlueRow getRowWithKey(IBlueRowCollection rows, string[] keyColumns, object[] colValues) { + foreach (IBlueRow r in rows) { object[] vals = getSelectedValues(r, keyColumns); if (CollectionHelpers.equal(vals, colValues)) return r; @@ -1204,8 +1077,7 @@ namespace BCG return null; } - public static IBlueRowCollection deepCopy(IBlueRowCollection rows) - { + public static IBlueRowCollection deepCopy(IBlueRowCollection rows) { BlueTable retVal = new BlueTable(rows.schema); foreach (IBlueRow r in rows) retVal.addRow(new BlueRow(r)); @@ -1219,8 +1091,7 @@ namespace BCG /// /// /// - public static int[] getColumnsInSortedOrder(IBlueTableSchema schema, bool includeId) - { + public static int[] getColumnsInSortedOrder(IBlueTableSchema schema, bool includeId) { ArrayList names = new ArrayList(schema.columnNames), retVal = new ArrayList(); names.Sort(); @@ -1233,40 +1104,33 @@ namespace BCG //this class is used so that we can sort a schema by column name //that way, prepublished tables will allways get published with schemas in the same order //regardless of how the underlying streams end up working - private class ColumnInfoComparisonWrapper : IComparable - { + private class ColumnInfoComparisonWrapper : IComparable { ColumnInfo myColumn; - public string ColumnName - { + public string ColumnName { get { return myColumn.colName; } //can't set it, these are only for reading columns } - public ColumnInfo Column - { + public ColumnInfo Column { get { return myColumn; } } - public ColumnInfoComparisonWrapper(ColumnInfo theColumn) - { + public ColumnInfoComparisonWrapper(ColumnInfo theColumn) { myColumn = theColumn; } - public int CompareTo(ColumnInfoComparisonWrapper otherGuy) - { + public int CompareTo(ColumnInfoComparisonWrapper otherGuy) { return ColumnName.CompareTo(otherGuy.ColumnName); } } - public static IBlueTableSchema getSortedSchema(IBlueTableSchema inSchema) - { + public static IBlueTableSchema getSortedSchema(IBlueTableSchema inSchema) { List sortedColNames = new List(inSchema.columnNames); sortedColNames.Sort(); List sortedColInfos = new List(); int colNo = 0; - foreach (string c in sortedColNames) - { + foreach (string c in sortedColNames) { ColumnInfo ci = inSchema[c]; sortedColInfos.Add(new ColumnInfo(c, colNo++, ci.colType)); @@ -1274,15 +1138,13 @@ namespace BCG return new BlueTableSchema(inSchema.tableName, sortedColInfos.ToArray()); } - public static IBlueTableSchema getSortedSchema(IBlueTableSchema theSchema, out int[] sortedSchemaToOriginalSchemaMapping, out int idColumnInSortedSchema) - { + public static IBlueTableSchema getSortedSchema(IBlueTableSchema theSchema, out int[] sortedSchemaToOriginalSchemaMapping, out int idColumnInSortedSchema) { //for now this function will not do anything //that is because we don't want to break the C++ deserialization code int numCols = theSchema.columnNames.Length; sortedSchemaToOriginalSchemaMapping = new int[numCols]; - for (int col = 0; col < theSchema.columnNames.Length; ++col) - { + for (int col = 0; col < theSchema.columnNames.Length; ++col) { //the mapping is just the identity mapping sortedSchemaToOriginalSchemaMapping[col] = col; @@ -1336,35 +1198,27 @@ namespace BCG /// /// /// true if equal - public static bool compareSchemas(IBlueTableSchema s1, IBlueTableSchema s2) - { + public static bool compareSchemas(IBlueTableSchema s1, IBlueTableSchema s2) { bool result = true; - try - { + try { if (s1.columnInfo.Length != s2.columnInfo.Length) result = false; else - foreach (ColumnInfo ci1 in s1.columnInfo) - { + foreach (ColumnInfo ci1 in s1.columnInfo) { ColumnInfo ci2 = s2[ci1.colNo]; - if (ci1.colName != ci2.colName || ci1.colType != ci2.colType || ci1.colNo != ci2.colNo) - { + if (ci1.colName != ci2.colName || ci1.colType != ci2.colType || ci1.colNo != ci2.colNo) { result = false; break; } } - } - catch - { + } catch { result = false; // the col# wasn't found in s2 } return result; } - public static void validate_row_schema(IBlueRow r) - { - foreach (var ci in r.schema.columnInfo) - { + public static void validate_row_schema(IBlueRow r) { + foreach (var ci in r.schema.columnInfo) { object rval = r[ci.colName]; if (rval == null) continue; @@ -1372,26 +1226,21 @@ namespace BCG } } - public static void validate_table_schema(IBlueRowCollection rows) - { - foreach (IBlueRow r in rows) - { + public static void validate_table_schema(IBlueRowCollection rows) { + foreach (IBlueRow r in rows) { validate_row_schema(r); } } - public static IBlueRowCollection removeRows(BlueTable table, string[] cols, object[] vals) - { + public static IBlueRowCollection removeRows(BlueTable table, string[] cols, object[] vals) { IBlueRowCollection dels = findMatchingRows(table, cols, vals); table.deleteRows(dels); return dels; } - public static IBlueRowCollection findMatchingRows(IBlueRowCollection rows, string[] cols, object[] vals) - { + public static IBlueRowCollection findMatchingRows(IBlueRowCollection rows, string[] cols, object[] vals) { BlueTable rv = new BlueTable(rows.schema); - foreach (IBlueRow r in rows) - { + foreach (IBlueRow r in rows) { bool match = true; for (int i = 0; i < cols.Length; i++) match &= vals[i].Equals(r[cols[i]]); @@ -1401,8 +1250,7 @@ namespace BCG return rv; } - public static void checkRowTypes(IBlueRowCollection rows) - { + public static void checkRowTypes(IBlueRowCollection rows) { foreach (IBlueRow r in rows) foreach (string col in r.schema.columnNames) if (r[col] != null && r[col] != DBNull.Value) @@ -1410,14 +1258,12 @@ namespace BCG String.Format("col {0} in table {1} is of type {2} in schema but is a {3} in the row", col, rows.schema.tableName, r.schema[col].colType.ToString(), r[col].GetType().ToString())); } - public static void addRows(ref BlueTable table, ICollection rows) - { + public static void addRows(ref BlueTable table, ICollection rows) { foreach (IBlueRow r in rows) table.addRow(r); } - public static void removeRows(ref BlueTable table, ICollection rows) - { + public static void removeRows(ref BlueTable table, ICollection rows) { foreach (IBlueRow r in rows) table.deleteRow(r); } @@ -1425,8 +1271,7 @@ namespace BCG public static void sortOutPacket(IBlueRowCollection dels, IBlueRowCollection adds, string[] keyCols, out IBlueRowCollection addedRows, out IBlueRowCollection deletedRows, - out IBlueRowCollection changedRows) - { + out IBlueRowCollection changedRows) { addedRows = new BlueTable(dels.schema); deletedRows = new BlueTable(dels.schema); changedRows = new BlueTable(dels.schema); @@ -1439,49 +1284,39 @@ namespace BCG foreach (IBlueRow r in adds) keyedAdds.update(r); - foreach (IBlueRow r in keyedDels) - { + foreach (IBlueRow r in keyedDels) { object[] key = BlueDataUtility.getSelectedValues(r, keyCols); - if (keyedAdds.getRow(key) != null) - { + if (keyedAdds.getRow(key) != null) { ///change (changedRows as BlueTable).addRow(r); - } - else - { + } else { ///delete (deletedRows as BlueTable).addRow(r); } } - foreach (IBlueRow r in keyedAdds) - { + foreach (IBlueRow r in keyedAdds) { object[] key = BlueDataUtility.getSelectedValues(r, keyCols); - if (keyedDels.getRow(key) == null) - { + if (keyedDels.getRow(key) == null) { (addedRows as BlueTable).addRow(r); } } } - public static int HashExpStrike(DateTime exp, double strike) - { + public static int HashExpStrike(DateTime exp, double strike) { return ((DateTime)exp).ToShortDateString().GetHashCode() ^ strike.GetHashCode(); } - public static IEnumerable collection_to_ienum(IBlueRowCollection rows) - { + public static IEnumerable collection_to_ienum(IBlueRowCollection rows) { List rv = new List(); foreach (IBlueRow r in rows) rv.Add(r); return rv; } - public static IBlueRowCollection ienum_to_collection(IEnumerable rows) - { + public static IBlueRowCollection ienum_to_collection(IEnumerable rows) { BlueTable rv = null; - if (rows.Count() > 0) - { + if (rows.Count() > 0) { rv = new BlueTable(rows.ElementAt(0).schema); foreach (IBlueRow r in rows) rv.addRow(r); @@ -1489,15 +1324,13 @@ namespace BCG return rv; } - public static IBlueTableSchema add_column(IBlueTableSchema input_schema, string new_col_name, Type new_col_type) - { + public static IBlueTableSchema add_column(IBlueTableSchema input_schema, string new_col_name, Type new_col_type) { List cis = new List(input_schema.columnInfo); cis.Add(new ColumnInfo(new_col_name, cis.Count, new_col_type)); return new BlueTableSchema(input_schema.tableName, cis.ToArray()); } - public static IBlueTableSchema remove_columns(IBlueTableSchema input_schema, params string[] col_names) - { + public static IBlueTableSchema remove_columns(IBlueTableSchema input_schema, params string[] col_names) { Dictionary removed_col_names = new Dictionary(); foreach (string c in col_names) removed_col_names[c] = true; @@ -1508,16 +1341,14 @@ namespace BCG return new BlueTableSchema(cis.ToArray(), input_schema.tableName); } - public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) - { + public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) { List rv = new List(); foreach (string c in col_names) rv.Add(schema[c].colNo); return rv.ToArray(); } - public static string get_appended_string_val(IBlueRow r, int[] col_indices) - { + public static string get_appended_string_val(IBlueRow r, int[] col_indices) { System.Text.StringBuilder sb = new System.Text.StringBuilder(); for (int i = 0; i < col_indices.Length; i++) sb.Append(r[col_indices[i]]); @@ -1525,35 +1356,29 @@ namespace BCG } public delegate string RowValueToString(IBlueRow r, int col_index); - public static string get_appended_string_val(IBlueRow r, int[] col_indices, RowValueToString to_string_conv) - { + public static string get_appended_string_val(IBlueRow r, int[] col_indices, RowValueToString to_string_conv) { System.Text.StringBuilder sb = new System.Text.StringBuilder(); for (int i = 0; i < col_indices.Length; i++) sb.Append(to_string_conv(r, i)); return sb.ToString(); } - public static IDictionary> group_together(IBlueRowCollection rows, string[] group_by_col_names) - { + public static IDictionary> group_together(IBlueRowCollection rows, string[] group_by_col_names) { BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length); return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher); } - public static IDictionary> group_together(IBlueRowCollection rows, string[] group_by_col_names, ref BlueRowHasher hasher) - { + public static IDictionary> group_together(IBlueRowCollection rows, string[] group_by_col_names, ref BlueRowHasher hasher) { return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher); } - public static IDictionary> group_together(IBlueRowCollection rows, int[] group_by_col_nos, ref BlueRowHasher hasher) - { + public static IDictionary> group_together(IBlueRowCollection rows, int[] group_by_col_nos, ref BlueRowHasher hasher) { Dictionary> rv = new Dictionary>(); hasher = hasher ?? new BlueRowHasher(group_by_col_nos.Length); - foreach (IBlueRow r in rows) - { + foreach (IBlueRow r in rows) { int row_hash = hasher.get_hash(r, group_by_col_nos); List this_group_rows = null; - if (!rv.TryGetValue(row_hash, out this_group_rows)) - { + if (!rv.TryGetValue(row_hash, out this_group_rows)) { this_group_rows = new List(); rv.Add(row_hash, this_group_rows); } @@ -1562,8 +1387,7 @@ namespace BCG return rv; } - public static void copy_to_row(IBlueRow row, string[] data_col_names, MutableBlueRow output_row) - { + public static void copy_to_row(IBlueRow row, string[] data_col_names, MutableBlueRow output_row) { foreach (string cn in data_col_names) output_row.setValue(cn, row[cn]); } @@ -1572,64 +1396,52 @@ namespace BCG /// Copies the column nos specified in data_cols from row, to output_row /// starting from index start_col_index /// - public static void copy_to_row(IBlueRow row, int[] data_col_nos, object[] output_row, int start_col_index) - { + public static void copy_to_row(IBlueRow row, int[] data_col_nos, object[] output_row, int start_col_index) { foreach (int col_no in data_col_nos) output_row[start_col_index++] = row[col_no]; } - public static object compute_min_max(object v1, object v2, bool is_max) - { - if (!TypeHelper.is_dbnull(v1) && !TypeHelper.is_dbnull(v2)) - { - if (TypeHelper.is_numeric_type(v1) && TypeHelper.is_numeric_type(v2)) - { + public static object compute_min_max(object v1, object v2, bool is_max) { + if (!TypeHelper.is_dbnull(v1) && !TypeHelper.is_dbnull(v2)) { + if (TypeHelper.is_numeric_type(v1) && TypeHelper.is_numeric_type(v2)) { double v1_num = (double)Convert.ChangeType(v1, TypeCode.Double), v2_num = (double)Convert.ChangeType(v2, TypeCode.Double); v1_num = Double.IsNaN(v1_num) ? (is_max ? Double.MinValue : Double.MaxValue) : v1_num; v2_num = Double.IsNaN(v2_num) ? (is_max ? Double.MinValue : Double.MaxValue) : v2_num; double rv = is_max ? Math.Max(v1_num, v2_num) : Math.Min(v1_num, v2_num); return Convert.ChangeType(rv, v1.GetType()); - } - else + } else return v1; - } - else + } else return TypeHelper.is_dbnull(v1) ? v2 : v1; } #region Sorting - public static class ParallelSort - { - public class TreeRowEffComparer : IComparer - { + public static class ParallelSort { + public class TreeRowEffComparer : IComparer { private int _depth_in_tree_col_no; private int[] _aggregation_tree_col_nos; private Pair[] _sort_map; - public TreeRowEffComparer(int depth_in_tree_col_no, int[] aggregation_tree_col_nos, Pair[] sort_map) - { + public TreeRowEffComparer(int depth_in_tree_col_no, int[] aggregation_tree_col_nos, Pair[] sort_map) { _depth_in_tree_col_no = depth_in_tree_col_no; _aggregation_tree_col_nos = aggregation_tree_col_nos; _sort_map = sort_map; } - public int Compare(IBlueRow x, IBlueRow y) - { + public int Compare(IBlueRow x, IBlueRow y) { return BlueDataUtility.sort_in_a_tree(x, y, _depth_in_tree_col_no, _aggregation_tree_col_nos, _sort_map); } } - public class TabularRowComparer : IComparer - { + public class TabularRowComparer : IComparer { private IEnumerable> _sort_map; public TabularRowComparer(IEnumerable> sort_map) { _sort_map = sort_map; } public int Compare(IBlueRow x, IBlueRow y) { return BlueDataUtility.compare_rows(x, y, _sort_map); } } - public class TabularRowHashPairComparer : IComparer> - { + public class TabularRowHashPairComparer : IComparer> { private Pair[] _sort_map; public TabularRowHashPairComparer(Pair[] sort_map) { _sort_map = sort_map; } @@ -1638,13 +1450,11 @@ namespace BCG #region Public Static Methods - public static void QuicksortSequential(IList arr, IComparer comparer) - { + public static void QuicksortSequential(IList arr, IComparer comparer) { QuicksortSequential(arr, comparer, 0, arr.Count - 1); } - public static void QuicksortParallel(IList arr, IComparer comparer) - { + public static void QuicksortParallel(IList arr, IComparer comparer) { QuicksortParallel(arr, comparer, 0, arr.Count - 1); } @@ -1652,27 +1462,20 @@ namespace BCG #region Private Static Methods - private static void QuicksortSequential(IList arr, IComparer comparer, int left, int right) - { - if (right > left) - { + private static void QuicksortSequential(IList arr, IComparer comparer, int left, int right) { + if (right > left) { int pivot = Partition(arr, comparer, left, right); QuicksortSequential(arr, comparer, left, pivot - 1); QuicksortSequential(arr, comparer, pivot + 1, right); } } - private static void QuicksortParallel(IList arr, IComparer comparer, int left, int right) - { + private static void QuicksortParallel(IList arr, IComparer comparer, int left, int right) { const int SEQUENTIAL_THRESHOLD = 64; - if (right > left) - { - if (right - left < SEQUENTIAL_THRESHOLD) - { + if (right > left) { + if (right - left < SEQUENTIAL_THRESHOLD) { QuicksortSequential(arr, comparer, left, right); - } - else - { + } else { int pivot = Partition(arr, comparer, left, right); System.Threading.Tasks.Parallel.Invoke(new Action[] { delegate {QuicksortParallel(arr, comparer, left, pivot - 1);}, delegate {QuicksortParallel(arr, comparer, pivot + 1, right); } @@ -1681,25 +1484,21 @@ namespace BCG } } - private static void Swap(IList arr, int i, int j) - { + private static void Swap(IList arr, int i, int j) { T tmp = arr[i]; arr[i] = arr[j]; arr[j] = tmp; } - private static int Partition(IList arr, IComparer comparer, int low, int high) - { + private static int Partition(IList arr, IComparer comparer, int low, int high) { // Simple partitioning implementation int pivotPos = (high + low) / 2; T pivot = arr[pivotPos]; Swap(arr, low, pivotPos); int left = low; - for (int i = low + 1; i <= high; i++) - { - if (comparer.Compare(arr[i], pivot) < 0) - { + for (int i = low + 1; i <= high; i++) { + if (comparer.Compare(arr[i], pivot) < 0) { left++; Swap(arr, i, left); } @@ -1712,10 +1511,8 @@ namespace BCG #endregion } - public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary sort_map) - { - foreach (KeyValuePair p in sort_map) - { + public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary sort_map) { + foreach (KeyValuePair p in sort_map) { int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value); if (comp != 0) return comp; @@ -1723,10 +1520,8 @@ namespace BCG return 0; } - public static int compare_rows(IBlueRow r1, IBlueRow r2, IEnumerable> sort_map) - { - foreach (Pair p in sort_map) - { + public static int compare_rows(IBlueRow r1, IBlueRow r2, IEnumerable> sort_map) { + foreach (Pair p in sort_map) { int comp = compare_send_dontcare_to_last(r1, p.first, r2, p.first, p.second); if (comp != 0) return comp; @@ -1736,15 +1531,13 @@ namespace BCG public static int compare_send_dontcare_to_last(IBlueRow r1, int r1_col_no, IBlueRow r2, int r2_col_no, - bool low_to_high) - { + bool low_to_high) { object o1 = BlueDataUtility.get_row_value(r1, r1_col_no, DBNull.Value), o2 = BlueDataUtility.get_row_value(r2, r2_col_no, DBNull.Value); return compare_send_dontcare_to_last(o1, o2, low_to_high); } - public static List> get_indexed_sort_map(List> sort_map, IBlueTableSchema schema) - { + public static List> get_indexed_sort_map(List> sort_map, IBlueTableSchema schema) { List> rv = new List>(); foreach (Pair p in sort_map) rv.Add(new Pair(schema[p.first].colNo, p.second)); @@ -1755,15 +1548,13 @@ namespace BCG /// empty strings etc are sent to the last public static int compare_send_dontcare_to_last(IBlueRow r1, string r1_col_name, IBlueRow r2, string r2_col_name, - bool low_to_high) - { + bool low_to_high) { object o1 = BlueDataUtility.get_row_value(r1, r1_col_name, DBNull.Value), o2 = BlueDataUtility.get_row_value(r2, r2_col_name, DBNull.Value); return compare_send_dontcare_to_last(o1, o2, low_to_high); } - public static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) - { + public static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) { if (is_dont_care_val(o1)) return is_dont_care_val(o2) ? 0 : 1; else @@ -1771,8 +1562,7 @@ namespace BCG : ((IComparable)o2).CompareTo(o1)); } - public static bool is_dont_care_val(object o) - { + public static bool is_dont_care_val(object o) { return o == null || o == DBNull.Value || o is string && String.IsNullOrEmpty((string)o) || o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) || @@ -1784,26 +1574,22 @@ namespace BCG #region Tree Aggregation - public static int[] convert_aggregation_tree(string[] aggregation_tree_col_names, IBlueTableSchema schema) - { + public static int[] convert_aggregation_tree(string[] aggregation_tree_col_names, IBlueTableSchema schema) { List rv = new List(); foreach (string col_name in aggregation_tree_col_names) rv.Add(schema[col_name].colNo); return rv.ToArray(); } - public static bool is_leaf_node(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) - { + public static bool is_leaf_node(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) { return BlueDataUtility.get_row_value(r, depth_in_tree_col_no, -1) == (aggregation_tree.Count() - 1); } - public static bool is_leaf_node(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) - { + public static bool is_leaf_node(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) { return BlueDataUtility.get_row_value(r, depth_in_tree_col_name, -1) == (aggregation_tree.Count() - 1); } - public static string generate_aggregation_based_id(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) - { + public static string generate_aggregation_based_id(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) { int aggregation_priority = BlueDataUtility.get_row_value(r, depth_in_tree_col_no, -1); string rv = String.Empty; for (int p = 0; p <= aggregation_priority; p++) @@ -1813,8 +1599,7 @@ namespace BCG /// Generates a string based id for a row in a tree /// by appending the values of the nodes above it - public static string generate_aggregation_based_id(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) - { + public static string generate_aggregation_based_id(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) { int aggregation_priority = BlueDataUtility.get_row_value(r, depth_in_tree_col_name, -1); string rv = String.Empty; for (int p = 0; p <= aggregation_priority; p++) @@ -1826,15 +1611,12 @@ namespace BCG /// Given an aggregation hierarchy tree, figures if the first /// arg is a parent of the second arg. /// - public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, string depth_in_tree_col_name, string[] aggregation_tree) - { + public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, string depth_in_tree_col_name, string[] aggregation_tree) { System.Diagnostics.Debug.Assert(parent_row.Contains(depth_in_tree_col_name) && child_row.Contains(depth_in_tree_col_name)); int parent_depth = BlueDataUtility.get_row_value(parent_row, depth_in_tree_col_name, Int32.MaxValue), child_depth = BlueDataUtility.get_row_value(child_row, depth_in_tree_col_name, Int32.MaxValue); - if (parent_depth < child_depth) - { - for (int p = 0; p <= parent_depth; p++) - { + if (parent_depth < child_depth) { + for (int p = 0; p <= parent_depth; p++) { string col_name = aggregation_tree.ElementAt(p); object parent_val = BlueDataUtility.get_row_value(parent_row, col_name, DBNull.Value), child_val = BlueDataUtility.get_row_value(child_row, col_name, DBNull.Value); @@ -1842,19 +1624,15 @@ namespace BCG return false; } return true; - } - else + } else return false; } - public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, int depth_in_tree_col_no, int[] aggregation_tree) - { + public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, int depth_in_tree_col_no, int[] aggregation_tree) { int parent_depth = BlueDataUtility.get_row_value(parent_row, depth_in_tree_col_no, Int32.MaxValue), child_depth = BlueDataUtility.get_row_value(child_row, depth_in_tree_col_no, Int32.MaxValue); - if (parent_depth < child_depth) - { - for (int p = 0; p <= parent_depth; p++) - { + if (parent_depth < child_depth) { + for (int p = 0; p <= parent_depth; p++) { int col_no = aggregation_tree[p]; object parent_val = BlueDataUtility.get_row_value(parent_row, col_no, DBNull.Value), child_val = BlueDataUtility.get_row_value(child_row, col_no, DBNull.Value); @@ -1862,26 +1640,21 @@ namespace BCG return false; } return true; - } - else + } else return false; } - public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, string depth_in_tree_col_name, string[] aggregation_tree, Dictionary sort_map) - { + public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, string depth_in_tree_col_name, string[] aggregation_tree, Dictionary sort_map) { if (BlueDataUtility.is_parent(r1, r2, depth_in_tree_col_name, aggregation_tree)) return -1; else if (BlueDataUtility.is_parent(r2, r1, depth_in_tree_col_name, aggregation_tree)) return 1; else if (get_row_value(r1, "id", Int32.MaxValue).Equals(get_row_value(r2, "id", Int32.MaxValue))) return 0; - else - { - for (int p = 0; p < aggregation_tree.Count(); p++) - { + else { + for (int p = 0; p < aggregation_tree.Count(); p++) { /// sort_map is used to leaf nodes - if (p == aggregation_tree.Count() - 1) - { + if (p == aggregation_tree.Count() - 1) { /// sorting for leaf nodes is special, since they can be /// sorted by data columns System.Diagnostics.Debug.Assert(BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree)); @@ -1901,21 +1674,17 @@ namespace BCG } } - public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, int depth_in_tree_col_no, int[] aggregation_tree, Pair[] sort_map) - { + public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, int depth_in_tree_col_no, int[] aggregation_tree, Pair[] sort_map) { if (BlueDataUtility.is_parent(r1, r2, depth_in_tree_col_no, aggregation_tree)) return -1; else if (BlueDataUtility.is_parent(r2, r1, depth_in_tree_col_no, aggregation_tree)) return 1; else if (get_row_value(r1, "id", Int32.MaxValue).Equals(get_row_value(r2, "id", Int32.MaxValue))) return 0; - else - { - for (int p = 0; p < aggregation_tree.Count(); p++) - { + else { + for (int p = 0; p < aggregation_tree.Count(); p++) { /// sort_map is used to leaf nodes - if (p == aggregation_tree.Count() - 1) - { + if (p == aggregation_tree.Count() - 1) { /// sorting for leaf nodes is special, since they can be /// sorted by data columns //System.Diagnostics.Debug.Assert(BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree)); @@ -1939,169 +1708,123 @@ namespace BCG #region Column accessors - public static T get_array_value(Array r, int col_no, T error_val) - { - try - { + public static T get_array_value(Array r, int col_no, T error_val) { + try { object rv = (r != null && col_no > -1 && col_no < r.Length) ? r.GetValue(col_no) : DBNull.Value; if (rv == null || rv == DBNull.Value) return error_val; else return (T)Convert.ChangeType(rv, typeof(T)); - } - catch - { + } catch { return error_val; } } - public static DateTime get_array_datetime_value(Array r, int col_no, DateTime error_val) - { - try - { + public static DateTime get_array_datetime_value(Array r, int col_no, DateTime error_val) { + try { object rv = (r != null && col_no > -1 && col_no < r.Length) ? r.GetValue(col_no) : error_val; return _get_datetime_value(rv, error_val); - } - catch - { + } catch { return error_val; } } - public static T get_row_value(IBlueRow r, int col_no, T error_val) - { - try - { + public static T get_row_value(IBlueRow r, int col_no, T error_val) { + try { object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : DBNull.Value; if (rv == null || rv == DBNull.Value) return error_val; else return (T)Convert.ChangeType(rv, typeof(T)); - } - catch - { + } catch { return error_val; } } - public static T get_row_value(IBlueRow r, string col_name, T error_val) - { - try - { + public static T get_row_value(IBlueRow r, string col_name, T error_val) { + try { object rv = (r != null && r.Contains(col_name)) ? r[col_name] : DBNull.Value; if (rv == null || rv == DBNull.Value) return error_val; else return (T)Convert.ChangeType(rv, typeof(T)); - } - catch - { + } catch { return error_val; } } - public static T get_row_enum_value(IBlueRow r, string col_name, T error_val) where T : struct - { - try - { + public static T get_row_enum_value(IBlueRow r, string col_name, T error_val) where T : struct { + try { object rv = r.Contains(col_name) ? r[col_name] : DBNull.Value; return _get_row_enum_value(rv, error_val); - } - catch - { + } catch { return error_val; } } - public static T get_row_enum_value(IBlueRow r, int col_no, T error_val) where T : struct - { - try - { + public static T get_row_enum_value(IBlueRow r, int col_no, T error_val) where T : struct { + try { object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : DBNull.Value; return _get_row_enum_value(rv, error_val); - } - catch - { + } catch { return error_val; } } - private static T _get_row_enum_value(object o, T error_val) where T : struct - { - try - { + private static T _get_row_enum_value(object o, T error_val) where T : struct { + try { if (o == null || o == DBNull.Value) return error_val; - else if (o is string) - { + else if (o is string) { T result = error_val; if (Enum.TryParse((string)o, true, out result)) return result; return error_val; - } - else if (o is Int32 || o is Int16 || o is UInt16 || o is UInt32) + } else if (o is Int32 || o is Int16 || o is UInt16 || o is UInt32) return (T)o; else if (o is T) return (T)o; else return error_val; - } - catch - { + } catch { return error_val; } } - public static DateTime get_row_datetime_value(IBlueRow r, string col_name, DateTime error_val) - { - try - { + public static DateTime get_row_datetime_value(IBlueRow r, string col_name, DateTime error_val) { + try { object rv = (r != null && r.Contains(col_name)) ? r[col_name] : error_val; return _get_datetime_value(rv, error_val); - } - catch - { + } catch { return error_val; } } - public static DateTime get_row_datetime_value(IBlueRow r, int col_no, DateTime error_val) - { - try - { + public static DateTime get_row_datetime_value(IBlueRow r, int col_no, DateTime error_val) { + try { object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : error_val; return _get_datetime_value(rv, error_val); - } - catch - { + } catch { return error_val; } } - private static DateTime _get_datetime_value(object v, DateTime error_val) - { - try - { + private static DateTime _get_datetime_value(object v, DateTime error_val) { + try { if (v == null || v == DBNull.Value) return error_val; - else - { + else { if (v is DateTime) return (DateTime)v; - else if (v is string) - { + else if (v is string) { DateTime rv_dt = error_val; DateTime.TryParse((string)v, out rv_dt); return rv_dt; - } - else - { + } else { double oa_date = (double)Convert.ChangeType(v, typeof(Double)); return DateTime.FromOADate(oa_date); } } - } - catch - { + } catch { return error_val; } } @@ -2112,8 +1835,7 @@ namespace BCG /// /// Reads a schema file name, and return the constructed schema /// - public static IBlueTableSchema createSchemaFromCSVSchemaFile(string schemaFileName) - { + public static IBlueTableSchema createSchemaFromCSVSchemaFile(string schemaFileName) { StreamReader sr = new StreamReader(schemaFileName); IBlueTableSchema retVal = createSchemaFromCSVSchemaFile(sr); sr.Close(); @@ -2124,8 +1846,7 @@ namespace BCG /// Builds a schema from a schema file. Note that the stream is not closed, so /// the calling code should do so /// - public static IBlueTableSchema createSchemaFromCSVSchemaFile(TextReader sr) - { + public static IBlueTableSchema createSchemaFromCSVSchemaFile(TextReader sr) { string tableName = sr.ReadLine(); ArrayList t = new ArrayList(); string l = ""; @@ -2133,14 +1854,12 @@ namespace BCG t.Add(l.Trim()); string[] cols = t.ToArray(typeof(string)) as String[]; ColumnInfo[] cis = new ColumnInfo[cols.Length]; - for (int i = 0; i < cols.Length; i++) - { + for (int i = 0; i < cols.Length; i++) { string[] colTags = cols[i].Split('/'); cis[i] = new ColumnInfo(colTags[0], i, TypeInstanceLoader.loadType(colTags[1])); } IBlueTableSchema retVal = new BlueTableSchema(cis, tableName); - if (retVal["id"] == null) - { + if (retVal["id"] == null) { ArrayList temp = new ArrayList(retVal.columnInfo); temp.Add(new ColumnInfo("id", temp.Count, typeof(System.Int32))); retVal = new BlueTableSchema(retVal.tableName, @@ -2153,17 +1872,13 @@ namespace BCG /// Creates an IBlueRow from a CSV data file row; if the string already contains an id, /// the id argument is ignored /// - public static IBlueRow createRowFromCSVFileRow(string fileRow, IBlueTableSchema schema, int id) - { + public static IBlueRow createRowFromCSVFileRow(string fileRow, IBlueTableSchema schema, int id) { string[] toks = CSVParser.parse(fileRow); ArrayList rowList = new ArrayList(); - if (toks.Length == schema.columnNames.Length) - { + if (toks.Length == schema.columnNames.Length) { for (int i = 0; i < toks.Length; i++) rowList.Add(myConvertTo(schema[i].colType, toks[i])); - } - else - { + } else { int idIndex = schema["id"].colNo; for (int i = 0; i < toks.Length; i++) rowList.Add(myConvertTo(schema[i < idIndex ? i : i + 1].colType, toks[i])); @@ -2175,8 +1890,7 @@ namespace BCG /// /// Returns the CSV data string corresponding to the row /// - public static string toCSVDataString(IBlueRow r) - { + public static string toCSVDataString(IBlueRow r) { string retVal = ""; for (int i = 0; i < r.schema.columnNames.Length; i++) retVal += @@ -2186,8 +1900,7 @@ namespace BCG return retVal.Remove(retVal.Length - 1, 1); } - public static string[] toCSVSchemaFileStrings(IBlueTableSchema schema) - { + public static string[] toCSVSchemaFileStrings(IBlueTableSchema schema) { string[] retVal = new string[2]; retVal[0] = schema.tableName; foreach (ColumnInfo ci in schema.columnInfo) @@ -2199,13 +1912,11 @@ namespace BCG /// /// Converts a CSV data value to its original type. /// - private static object myConvertTo(Type t, string val) - { + private static object myConvertTo(Type t, string val) { val = val.Trim(); if (t.Equals(typeof(System.String))) return val; - else if (val != "") - { + else if (val != "") { if (t.Equals(typeof(System.Char))) return Convert.ToChar(val); if (t.Equals(typeof(System.Int16))) @@ -2224,21 +1935,17 @@ namespace BCG return DBNull.Value; } - private static object myToDateTime(string val) - { - try - { + private static object myToDateTime(string val) { + try { return Convert.ToDateTime(val); - } - catch (FormatException) //datetime is not stored in one of the string formats - { + } catch (FormatException) //datetime is not stored in one of the string formats + { return val.IndexOf('.') > -1 ? DateTime.FromOADate(Convert.ToDouble(val)) : new DateTime(Convert.ToInt64(val)); } } - public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) - { + public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) { StreamReader sr = new StreamReader(fileAddress); string cols = sr.ReadLine(); string[] columns = cols.Split(','); @@ -2261,18 +1968,15 @@ namespace BCG /// /// Uses a cached compiled expression to implement a truly generic add function. Will throw if given types that cannot be added. /// - internal class MagicAdder - { + internal class MagicAdder { static Func add; - static MagicAdder() - { + static MagicAdder() { ParameterExpression left = Expression.Parameter(typeof(T), "a"); ParameterExpression right = Expression.Parameter(typeof(T), "b"); add = (Func)Expression.Lambda(Expression.Add(left, right), left, right).Compile(); } - public static T Add(T a, T b) - { + public static T Add(T a, T b) { return add(a, b); } } @@ -2280,8 +1984,7 @@ namespace BCG #region Misc - public static void swap(ref T t1, ref T t2) - { + public static void swap(ref T t1, ref T t2) { T temp = t1; t1 = t2; t2 = temp; diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs index 9407a102f..bc5e2e974 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Program.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs @@ -11,7 +11,7 @@ using System.Threading; bts.Log.Gen.info($"Starting DSPublisher"); Console.WriteLine("Starting DSPublisher..."); -bool run_echo_tcp_client_only = true; +bool run_echo_tcp_client_only = false; static void init(string[] args) { Func login = async (l, p) => { var login = l; @@ -31,6 +31,7 @@ else static ViewStream build_stream(string stream_expression) { try { + bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}"); var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); var rv = exp.getView(new Dictionary(0)); return rv; @@ -62,10 +63,12 @@ void _publish_streams() { {"instruments","instruments()"}, //{"timesales","timesales(10000)"}, {"books","books()"}, - // {"seconds_1_10","second(1,10)"}, + {"seconds_1_10","second(1,10)"}, {"literal_420","literal(\"x\", \"420.0\")"}, - // {"bbo_ts","timeSeries(bbo())"} - }; + //{"mongodb_testcollection", "mongodb(\"mongodb://192.168.1.80:27017\",\"local\",\"test_collection\")"} + }; + // {"bbo_ts","timeSeries(bbo())"} + foreach (var kv in streams_to_publish) { var name = kv.Key; var exp = kv.Value; @@ -77,6 +80,7 @@ void _publish_streams() { System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1)); ViewStreamDatabase.instance.doWork(); csm.doWork(); + MasterWorker.getInstance(false).doWork(); } } diff --git a/src/DataStreamSuite/PubSub/Client/Communication/IPubSubServerConnection.cs b/src/DataStreamSuite/PubSub/Client/Communication/IPubSubServerConnection.cs index 132248260..1ec1ab957 100644 --- a/src/DataStreamSuite/PubSub/Client/Communication/IPubSubServerConnection.cs +++ b/src/DataStreamSuite/PubSub/Client/Communication/IPubSubServerConnection.cs @@ -47,30 +47,30 @@ namespace BCG { /// of Int32.Max would return all the messages it has /// received between last getUpdates and this call. /// getUpdates(TimeSpan timeout, int maxElements); + IList get_updates(TimeSpan timeout, int maxElements); /// /// server URL to which the connection is made /// - string serverURL { get; } + string server_url { get; } /// /// client URL as seen by the server. if a connection has not /// been established (as can happen in TCP) an empty string /// is returned /// - string clientURL { get; } + string client_url { get; } /// /// registers the callback to receive /// connection notifications /// - void registerCallback(ConnectionStatusChange del); + void register_connection_callback(PubSubServerConnectionStatusChange del); /// /// registers callbacks to reveive data /// /// - void registerDataCallback(DataDelegate del); + void register_data_callback(PubSubServerDataDelegate del); - void registerSendConfirmationCallback(SendConfirmationDelegate del); + void register_send_confirmation_callback(SendConfirmationDelegate del); } } diff --git a/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs b/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs index 68306d289..4cc816364 100644 --- a/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs +++ b/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs @@ -8,6 +8,7 @@ using System.IO; //using BCG.Utilities.General; using System.Threading; using mbts; +using bts; namespace BCG { /// @@ -20,26 +21,27 @@ namespace BCG { /// ITCPConnectionHandler.onConnection will be called on any arbitrary thread /// /// - public class PubSubServerManagedTCPConnection : IPubSubServerConnection, - ITCPMessageHandler, - ITCPConnectionHandler { + public class PubSubServerManagedTCPConnection : IPubSubServerConnection,// external facing interface to communicate msgs to Pubsubserver + ITCPMessageHandler, // to get messages from ITCPCOnnection + ITCPConnectionHandler { // to get connections from AsyncTCPConnector_APM private ITCPConnection _connection, _new_connection; private readonly BlutoSubAddress _server_address; // if a message(with confirm) can not be sent , its id is queued here, // on reconnection, _on_send_onfirm_callback is fired with a negative on these ids, so the user //can re-send it. The re-send mechanism is not in implemented in this class, but by the user of this class private ThreadSafeQueue _negative_unsent_confs = new ThreadSafeQueue(); - // in onData, data is queued here + // in onData, data is queued here, for polling private SyncQueue _in_data_queue = new SyncQueue(); private DateTime _last_hb = DateTime.Now; - private ConnectionStatusChange _conn_callback; - private DataDelegate _data_callback; - private SendConfirmationDelegate _on_send_onfirm_callback; + private PubSubServerConnectionStatusChange _conn_callback; + private PubSubServerDataDelegate _data_callback; + private SendConfirmationDelegate _on_send_confirm_callback; private readonly TimeSpan _connection_drop_threshold = - new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval * PubSub3Config.instance.PS3_MissingHeartBeatsForGC); + new TimeSpan(0, 0, 0, 0, + PubSub3Config.instance.PS3_HeartBeatInterval * PubSub3Config.instance.PS3_MissingHeartBeatsForGC); - public PubSubServerManagedTCPConnection(string serverURL) - : this(new BlutoSubAddress(serverURL)) { } + public PubSubServerManagedTCPConnection(string server_url) + : this(new BlutoSubAddress(server_url)) { } public PubSubServerManagedTCPConnection(BlutoSubAddress server_address) { _server_address = server_address; @@ -48,17 +50,17 @@ namespace BCG { private void _connect_to_ps_server() { bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server: {_server_address}"); - ClientToServerManagedTCPConnector.connect_to(_server_address, this, TimeSpan.FromSeconds(1)); + AsyncTCPConnector_APM.connect_to(_server_address, this); } #region IPubSubServerConnection Members - public void sendWithConfirmation(byte[] data, ref string conf_id) { - bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} myConnection : {_connection != null}"); + public void send_with_confirmation(byte[] data, ref string conf_id) { + bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} _connection : {_connection != null}"); if (_connection != null) { _connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) => { - if (_on_send_onfirm_callback != null) - _on_send_onfirm_callback(msg_id, confirmed); + if (_on_send_confirm_callback != null) + _on_send_confirm_callback(msg_id, confirmed); }); } else { bts.Log.Gen.warn($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} no connection, buffering send"); @@ -86,18 +88,18 @@ namespace BCG { public void send_with_confirmation(PubSubMessage msg, ref string conf_id) { var bytes = msg.to_bytes(); - sendWithConfirmation(bytes, ref conf_id); + send_with_confirmation(bytes, ref conf_id); } - public IList getUpdates(TimeSpan timeout, int maxElements) { + public IList get_updates(TimeSpan timeout, int maxElements) { return _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements); } - public string serverURL { + public string server_url { get { return _server_address.ToString(); } } - public string clientURL { + public string client_url { get { string rv = ""; lock (this) { @@ -118,16 +120,16 @@ namespace BCG { } } - public void registerCallback(ConnectionStatusChange del) { + public void register_connection_callback(PubSubServerConnectionStatusChange del) { _conn_callback = del; } - public void registerDataCallback(DataDelegate del) { + public void register_data_callback(PubSubServerDataDelegate del) { _data_callback = del; } - public void registerSendConfirmationCallback(SendConfirmationDelegate del) { - _on_send_onfirm_callback = del; + public void register_send_confirmation_callback(SendConfirmationDelegate del) { + _on_send_confirm_callback = del; } #endregion @@ -185,18 +187,6 @@ namespace BCG { break; } - // this stupid shit, caused duplicating og messages and 2 days of wild goose chase - // if (message.messageType == RemoteMessageType.Data) - // { - // if (_data_callback != null) - // _data_callback(message.remoteConnectionInfo.ToString(), message.message); - // else - // _in_data_queue.enqueue(message.message); - // } - // else - // { - // _last_hb = DateTime.Now; - // } } #endregion @@ -204,8 +194,7 @@ namespace BCG { #region ITCPConnectionHandler Members /// - /// This function is called on the main thread (as all other functions - /// except onData) + /// will be called on ana rbitrary .Net thread /// /// public void onConnection(ITCPConnection connection) { @@ -246,10 +235,10 @@ namespace BCG { _connection.register(this); if (_conn_callback != null) _conn_callback(_server_address.ToString(), true); - if (_on_send_onfirm_callback != null) { + if (_on_send_confirm_callback != null) { var msg_ids = _negative_unsent_confs.dequeueAll(); foreach (var id in msg_ids) - _on_send_onfirm_callback(id, false); + _on_send_confirm_callback(id, false); } } } diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs index 495bb143b..520869a3c 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs @@ -328,7 +328,8 @@ namespace bts { return; } _send_key_map.Add(message.key); - bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: pre endSend this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} message: {res.AsyncState} _is_alive: {_is_alive}"); + bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: pre endSend this: {ToString()} tid: {_tid()} " + + $"res: {_cb_txt(res)} message: {res.AsyncState} _is_alive: {_is_alive}"); if (_is_alive) { try { SocketError send_err; diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/ClientToServerManagedTCPConnector.cs b/src/DataStreamSuite/PubSub/ManagedTCP/ClientToServerManagedTCPConnector.cs index 3b987a8d9..b0115ec00 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/ClientToServerManagedTCPConnector.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/ClientToServerManagedTCPConnector.cs @@ -1,82 +1,82 @@ -using System; -using System.Collections.Generic; -using System.Text; -using System.Net; -using System.Net.Sockets; -using System.Threading; -using System.Threading.Tasks; -//using BCG.Utilities; -//using BCG.Utilities.General; +// using System; +// using System.Collections.Generic; +// using System.Text; +// using System.Net; +// using System.Net.Sockets; +// using System.Threading; +// using System.Threading.Tasks; +// //using BCG.Utilities; +// //using BCG.Utilities.General; -namespace BCG { - internal class SocketConnectionArgs { - public EndPoint ep; - public Socket sock; - public ITCPConnectionHandler handler; - public string serverURL; +// namespace BCG { +// internal class SocketConnectionArgs { +// public EndPoint ep; +// public Socket sock; +// public ITCPConnectionHandler handler; +// public string serverURL; - internal SocketConnectionArgs(EndPoint theEp, Socket theSock, ITCPConnectionHandler theHandler, - string theServerURL) { - ep = theEp; - sock = theSock; - handler = theHandler; - serverURL = theServerURL; - } - } +// internal SocketConnectionArgs(EndPoint theEp, Socket theSock, ITCPConnectionHandler theHandler, +// string theServerURL) { +// ep = theEp; +// sock = theSock; +// handler = theHandler; +// serverURL = theServerURL; +// } +// } - public class ClientToServerManagedTCPConnector { - private static Random myRnd = new Random(DateTime.Now.Millisecond); - private static TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval); - private static int myWaitTimeBeforeRetryingUpperLimit = 5000, - myWaitTimeBeforeRetryingLowerLimit = 1000; +// public class ClientToServerManagedTCPConnector { +// private static Random myRnd = new Random(DateTime.Now.Millisecond); +// private static TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval); +// private static int myWaitTimeBeforeRetryingUpperLimit = 5000, +// myWaitTimeBeforeRetryingLowerLimit = 1000; - public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler) { - Console.WriteLine($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}"); - Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); - EndPoint ep = new IPEndPoint(IPAddress.Parse(serverAddress.ip), serverAddress.port); - sock.BeginConnect(ep, new AsyncCallback(myOnConnect), new SocketConnectionArgs(ep, sock, handler, serverAddress.ToString())); - } +// public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler) { +// Console.WriteLine($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}"); +// Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); +// EndPoint ep = new IPEndPoint(IPAddress.Parse(serverAddress.ip), serverAddress.port); +// sock.BeginConnect(ep, new AsyncCallback(myOnConnect), new SocketConnectionArgs(ep, sock, handler, serverAddress.ToString())); +// } - public static async Task connect_to_async(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) { - bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}"); - await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts); - } +// public static async Task connect_to_async(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) { +// bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}"); +// await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts); +// } - public static void connect_to(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) { - bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}"); - Task.Run(async () => { - await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts); - }); - } +// public static void connect_to(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) { +// bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}"); +// Task.Run(async () => { +// await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts); +// }); +// } - private static void myOnConnect(IAsyncResult res) { - Console.WriteLine("ClientToServerManagedTCPConnector.myOnConnect " + res.ToString() + " " + res.AsyncState.ToString()); - SocketConnectionArgs args = res.AsyncState as SocketConnectionArgs; - try { - args.sock.EndConnect(res); - } catch (SocketException sex) { - string errMsg = String.Format("{0}: Connection to {1} failed.", DateTime.Now.ToString(), args.serverURL.ToString()); - //BlueEventLog.logEvent(sex.Message + ";" + errMsg, PS3LogLevels.disconnection); - //Console.WriteLine(errMsg); - if (args.handler.isDisposed) { - string doneMsg = String.Format("{0}: Connection to {1} was disposed. Not Retrying.", DateTime.Now.ToString(), args.serverURL.ToString()); - //Console.WriteLine(doneMsg); - // BlueEventLog.logEvent(doneMsg, PS3LogLevels.disconnection); - return; - } else { - string retryMsg = String.Format("{0}: Retrying Connection to {1}.", DateTime.Now.ToString(), args.serverURL.ToString()); - //Console.WriteLine(retryMsg); - //BlueEventLog.logEvent(retryMsg, PS3LogLevels.disconnection); +// private static void myOnConnect(IAsyncResult res) { +// Console.WriteLine("ClientToServerManagedTCPConnector.myOnConnect " + res.ToString() + " " + res.AsyncState.ToString()); +// SocketConnectionArgs args = res.AsyncState as SocketConnectionArgs; +// try { +// args.sock.EndConnect(res); +// } catch (SocketException sex) { +// string errMsg = String.Format("{0}: Connection to {1} failed.", DateTime.Now.ToString(), args.serverURL.ToString()); +// //BlueEventLog.logEvent(sex.Message + ";" + errMsg, PS3LogLevels.disconnection); +// //Console.WriteLine(errMsg); +// if (args.handler.isDisposed) { +// string doneMsg = String.Format("{0}: Connection to {1} was disposed. Not Retrying.", DateTime.Now.ToString(), args.serverURL.ToString()); +// //Console.WriteLine(doneMsg); +// // BlueEventLog.logEvent(doneMsg, PS3LogLevels.disconnection); +// return; +// } else { +// string retryMsg = String.Format("{0}: Retrying Connection to {1}.", DateTime.Now.ToString(), args.serverURL.ToString()); +// //Console.WriteLine(retryMsg); +// //BlueEventLog.logEvent(retryMsg, PS3LogLevels.disconnection); - int waitTime = myRnd.Next(myWaitTimeBeforeRetryingLowerLimit, myWaitTimeBeforeRetryingUpperLimit); - Thread.Sleep(waitTime); - args.sock.BeginConnect(args.ep, new AsyncCallback(myOnConnect), args); - return; - } - } - //BlueEventLog.logEvent(String.Format("Successful connection made to {0}.", args.serverURL.ToString()), PS3LogLevels.connection); - args.handler.onConnection(new bts.AsyncTCPConnection_APM(args.sock, myHBTimeSpan, ITCPConnection.ConnectionSide.Client)); - } +// int waitTime = myRnd.Next(myWaitTimeBeforeRetryingLowerLimit, myWaitTimeBeforeRetryingUpperLimit); +// Thread.Sleep(waitTime); +// args.sock.BeginConnect(args.ep, new AsyncCallback(myOnConnect), args); +// return; +// } +// } +// //BlueEventLog.logEvent(String.Format("Successful connection made to {0}.", args.serverURL.ToString()), PS3LogLevels.connection); +// args.handler.onConnection(new bts.AsyncTCPConnection_APM(args.sock, myHBTimeSpan, ITCPConnection.ConnectionSide.Client)); +// } - } -} +// } +// } diff --git a/src/DataStreamSuite/PubSub/Server/Communication/IPubsubMessageHubBase.cs b/src/DataStreamSuite/PubSub/Server/Communication/IPubsubMessageHubBase.cs index 9fbbb852e..8e7bfc15b 100644 --- a/src/DataStreamSuite/PubSub/Server/Communication/IPubsubMessageHubBase.cs +++ b/src/DataStreamSuite/PubSub/Server/Communication/IPubsubMessageHubBase.cs @@ -5,18 +5,16 @@ using System.IO; using System.Collections.Generic; using mbts; -namespace BCG -{ - public delegate void ConnectionStatusChange(string peerURL, bool connection); - public delegate void DataDelegate(string sender, byte[] data); +namespace BCG { + public delegate void PubSubServerConnectionStatusChange(string peerURL, bool connection); + public delegate void PubSubServerDataDelegate(string sender, byte[] data); /// /// Encapsulates PubsubServer's communication with the /// rest of the world. Has send and get methods, and also /// an event to tell if a client connects or disconnects. /// this event is prompted by the doWork. /// - public interface IPubsubMessageHub : IWorker - { + public interface IPubsubMessageHub : IWorker { /// /// The port that the message hub is listening on. Return -1 if you're not listening on a port, i.e. for hte case of multicast @@ -29,14 +27,14 @@ namespace BCG /// getUpdates. The delegate will be fired on an arbitrary thread. /// /// - void registerDataCallback(DataDelegate del); + void registerDataCallback(PubSubServerDataDelegate del); /// /// registers a connection callback. this callback /// will be fired within the doWork call. /// /// /// - void registerConnectionCallback(ConnectionStatusChange del); + void registerConnectionCallback(PubSubServerConnectionStatusChange del); /// /// this is where this interface starts communication with the /// rest of the world. Obvioulsy, the resgiters should be diff --git a/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs b/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs index a8acd1250..b2e2518d9 100644 --- a/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs +++ b/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs @@ -22,8 +22,8 @@ namespace BCG { } private bool myActivated = false; - private ConnectionStatusChange myConnCb; - private DataDelegate myDataCb; + private PubSubServerConnectionStatusChange myConnCb; + private PubSubServerDataDelegate myDataCb; /// /// client to connection @@ -77,12 +77,12 @@ namespace BCG { #region IPubsubMessageHub Members - public void registerDataCallback(DataDelegate del) { + public void registerDataCallback(PubSubServerDataDelegate del) { // BlueEventLog.logAssert(!myActivated, "connection callbacks should be registered before the call to activate"); myDataCb = del; } - public void registerConnectionCallback(ConnectionStatusChange del) { + public void registerConnectionCallback(PubSubServerConnectionStatusChange del) { //BlueEventLog.logAssert(!myActivated, "data callbacks should be registered before the call to activate"); myConnCb = del; } diff --git a/src/DataStreamSuite/PubSub/Server/Communication/ServerMessageHub.cs b/src/DataStreamSuite/PubSub/Server/Communication/ServerMessageHub.cs index 7a8fb319b..af1301f19 100644 --- a/src/DataStreamSuite/PubSub/Server/Communication/ServerMessageHub.cs +++ b/src/DataStreamSuite/PubSub/Server/Communication/ServerMessageHub.cs @@ -4,25 +4,22 @@ using System.Threading; using System.IO; using mbts; -namespace BCG -{ +namespace BCG { /// /// Wraps around all the IPubsubMessageHub's that pubsub supports. /// ie MSMQ and TCP to begin with. /// - public class ServerMessageHub : IPubsubMessageHub - { + public class ServerMessageHub : IPubsubMessageHub { private static ServerMessageHub mySelf = null; private static object myInstanceLock = new object(); private Dictionary myHubs = new Dictionary(); private SyncQueue> myReceivingQueue = new SyncQueue>(); - private ConnectionStatusChange myConnectionCb; - private DataDelegate myDataCb; + private PubSubServerConnectionStatusChange myConnectionCb; + private PubSubServerDataDelegate myDataCb; private MasterWorker myMasterWorker = MasterWorker.getInstance(false); protected ServerMessageHub() : this(PubSub3Config.instance.PS3_ServerPort) { } - protected ServerMessageHub(int portNumber) - { + protected ServerMessageHub(int portNumber) { // TODO: no more MSMQ // if (!PubSub3Config.instance.PS3_DisableMSMQ) // myHubs.Add(MSMQSubAddress.prefix, new MSMQMessageHub()); @@ -35,19 +32,14 @@ namespace BCG as IPubsubMessageHub); } - public int port - { + public int port { get { return myHubs[BlutoSubAddress.prefix].port; } } - public static ServerMessageHub instance - { - get - { - if (mySelf == null) - { - lock (myInstanceLock) - { + public static ServerMessageHub instance { + get { + if (mySelf == null) { + lock (myInstanceLock) { if (mySelf == null) mySelf = new ServerMessageHub(); } @@ -60,47 +52,39 @@ namespace BCG /* Builds the server message hub with a specified port number. Only works if the server message hub has not already been created. * */ - public static ServerMessageHub getInstance(int portNumber) - { - if (mySelf == null) - { - lock (myInstanceLock) - { + public static ServerMessageHub getInstance(int portNumber) { + if (mySelf == null) { + lock (myInstanceLock) { if (mySelf == null) mySelf = new ServerMessageHub(portNumber); } - } - else + } else throw new ArgumentException("Error: Cannot specify a port number for the ServerMessageHub instance if it has already been created."); return mySelf; } #region IPubsubMessageHub Members - public void registerDataCallback(DataDelegate del) { myDataCb = del; } + public void registerDataCallback(PubSubServerDataDelegate del) { myDataCb = del; } - public void registerConnectionCallback(ConnectionStatusChange del) { myConnectionCb = del; } + public void registerConnectionCallback(PubSubServerConnectionStatusChange del) { myConnectionCb = del; } - public void activate() - { + public void activate() { string msg = "PubSubServer started .. initializing TCP transport components .. "; Console.WriteLine(msg); - foreach (IPubsubMessageHub hub in myHubs.Values) - { - hub.registerDataCallback(new DataDelegate(myOnData)); - hub.registerConnectionCallback(new ConnectionStatusChange(myOnConnection)); + foreach (IPubsubMessageHub hub in myHubs.Values) { + hub.registerDataCallback(new PubSubServerDataDelegate(myOnData)); + hub.registerConnectionCallback(new PubSubServerConnectionStatusChange(myOnConnection)); hub.activate(); } myMasterWorker.addWorker(this, new TimeSpan(0, 0, 0, 0, 250)); } - public List> getUpdates(TimeSpan timeout, int maxElements) - { + public List> getUpdates(TimeSpan timeout, int maxElements) { return myReceivingQueue.dequeueMultipleElements(timeout, maxElements); } - public bool send(string receiver, byte[] data) - { + public bool send(string receiver, byte[] data) { if (receiver.StartsWith(BlutoSubAddress.prefix)) return myHubs[BlutoSubAddress.prefix].send(receiver, data); else if (receiver.StartsWith(MSMQSubAddress.prefix)) @@ -109,8 +93,7 @@ namespace BCG return false; } - public bool send(string receiver, Stream str) - { + public bool send(string receiver, Stream str) { if (receiver.StartsWith(BlutoSubAddress.prefix)) return myHubs[BlutoSubAddress.prefix].send(receiver, str); else if (receiver.StartsWith(MSMQSubAddress.prefix)) @@ -126,8 +109,7 @@ namespace BCG /// /// connection callback will be called within this doWork /// - public void doWork() - { + public void doWork() { foreach (IPubsubMessageHub hub in myHubs.Values) hub.doWork(); } @@ -138,21 +120,18 @@ namespace BCG public static string internalPublisherConnection = "InternalPublisher"; - public void internalSend(byte[] data) - { + public void internalSend(byte[] data) { myReceivingQueue.enqueue(new Pair(internalPublisherConnection, data)); } - private void myOnData(string sender, byte[] msg) - { + private void myOnData(string sender, byte[] msg) { if (myDataCb != null) myOnData(sender, msg); else myReceivingQueue.enqueue(new Pair(sender, msg)); } - private void myOnConnection(string clientURL, bool connected) - { + private void myOnConnection(string clientURL, bool connected) { if (myConnectionCb != null) myConnectionCb(clientURL, connected); } diff --git a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs index 35460d027..749c3437b 100644 --- a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs +++ b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs @@ -58,7 +58,7 @@ namespace BCG { myReceivingProcThread = new Thread(new ThreadStart(myReceivingThreadProc)); myMessageHub = ServerMessageHub.instance; - myMessageHub.registerConnectionCallback(new ConnectionStatusChange(myHandleConnections)); + myMessageHub.registerConnectionCallback(new PubSubServerConnectionStatusChange(myHandleConnections)); } diff --git a/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs b/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs index 0fa8a86f7..32bb09357 100644 --- a/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs +++ b/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs @@ -33,7 +33,7 @@ namespace BCG { public SingleServerSubscriberCore(string serverURL, PubSubServerConnectionFactory connectionFactory) { myConnectionFactory = connectionFactory; myConnection = myConnectionFactory.getConnection(serverURL); - myConnection.registerCallback(new ConnectionStatusChange(myHandleServerReboot)); + myConnection.register_connection_callback(new PubSubServerConnectionStatusChange(myHandleServerReboot)); } public IList getUpdates() { @@ -49,7 +49,7 @@ namespace BCG { public IList getUpdates(int timeout) { myConnection.doWork(); List retVal = new List(); - IList dataItems = myConnection.getUpdates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue); + IList dataItems = myConnection.get_updates(new TimeSpan(0, 0, 0, 0, timeout), Int32.MaxValue); foreach (byte[] data in dataItems) { processMessage(data, retVal); } @@ -65,7 +65,7 @@ namespace BCG { foreach (string tag in tags) { if (myTags.Contains(tag)) { string errMsg = String.Format("tag {0} was already subscribed from server {1}", - tag, myConnection.serverURL); + tag, myConnection.server_url); // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning); } else { myTags.Add(tag); @@ -82,11 +82,11 @@ namespace BCG { unsubTags.Add(tag); } else { string errMsg = String.Format("tag {0} is being unsubscribed from server {1}, eventhough " + - "it was never subscribed", tag, myConnection.serverURL); + "it was never subscribed", tag, myConnection.server_url); // BlueEventLog.logEvent(errMsg, BlueEventLevel.Warning); } } - UnSubMetaData unsub = new UnSubMetaData(myConnection.clientURL, unsubTags.ToArray()); + UnSubMetaData unsub = new UnSubMetaData(myConnection.client_url, unsubTags.ToArray()); Stream str = new MemoryStream(); new MetaData(unsub).save(str); str.Position = 0; @@ -109,7 +109,7 @@ namespace BCG { #endregion private void mySubscribe(params string[] tags) { - SubMetaData sub = new SubMetaData(myConnection.clientURL, tags); + SubMetaData sub = new SubMetaData(myConnection.client_url, tags); Stream str = new MemoryStream(); new MetaData(sub).save(str); str.Position = 0; diff --git a/src/DataStreamSuite/PubSub/StaticPublisherCore.cs b/src/DataStreamSuite/PubSub/StaticPublisherCore.cs index 41352f3af..48156debe 100644 --- a/src/DataStreamSuite/PubSub/StaticPublisherCore.cs +++ b/src/DataStreamSuite/PubSub/StaticPublisherCore.cs @@ -387,18 +387,18 @@ namespace BCG { SharpZipCompressor Kompressor = new SharpZipCompressor(); Kompressor.compress(content, contentToPublish); contentToPublish.Position = 0; - pub_md = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString); + pub_md = new PubMetaData(tagType, tag, (int)contentToPublish.Length, (int)content.Length, serverConn.server_url, ClientInfo.thisClient.processString); if (PubSub3Config.instance.shouldReportTagsCompressionRatio(tag)) { string msg = String.Format("Compressed tag {0} from {1} to {2} bytes, for a ratio of {3}.", tag, content.Length, contentToPublish.Length, (float)content.Length / (float)contentToPublish.Length); // BlueEventLog.logEvent(msg, PS3LogLevels.compressionRatio); } } else - pub_md = new PubMetaData(tagType, tag, (int)content.Length, serverConn.serverURL, ClientInfo.thisClient.processString); + pub_md = new PubMetaData(tagType, tag, (int)content.Length, serverConn.server_url, ClientInfo.thisClient.processString); content.Position = initPos; if (CrushedTagManager.instance.is_tag_crushable(tag)) { string refId = String.Empty; - Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.serverURL); + Console.WriteLine("StaticPublisherCore.mySend serverConn.sendWithConfirmation tag: " + tag + " server: " + serverConn.server_url); // this is where the data is actually sent out (there are multiple sends) PubSubMessage ps_msg = new PubSubMessage(new MetaData(pub_md), contentToPublish); //Console.WriteLine($"serverConn.send_with_confirmation metadata: {ps_msg.meta_data}"); @@ -428,8 +428,8 @@ namespace BCG { myTagToSender.Add(tag, sender); if (!mySenders.Contains(sender)) { mySenders.Add(sender); - sender.registerCallback(new ConnectionStatusChange(myHandleServerReboot)); - sender.registerSendConfirmationCallback(new SendConfirmationDelegate(myOnSendConfirm)); + sender.register_connection_callback(new PubSubServerConnectionStatusChange(myHandleServerReboot)); + sender.register_send_confirmation_callback(new SendConfirmationDelegate(myOnSendConfirm)); } } @@ -443,7 +443,7 @@ namespace BCG { if (connection) { string thisPublisherInfo = ClientInfo.thisClient.processString; foreach (KeyValuePair p in myTagToSender) { - if (p.Value.serverURL == serverURL) { + if (p.Value.server_url == serverURL) { string tagType; string tag = p.Key; myCrushableTagsWaitingOnAck.Clear(); diff --git a/src/DataStreamSuite/PubSubServer/Program.cs b/src/DataStreamSuite/PubSubServer/Program.cs index d98cc68ac..39fac8009 100644 --- a/src/DataStreamSuite/PubSubServer/Program.cs +++ b/src/DataStreamSuite/PubSubServer/Program.cs @@ -15,7 +15,7 @@ namespace BCG { /// [STAThread] public static void Main(string[] args) { - bool run_echo_tcp_server_only = true; + bool run_echo_tcp_server_only = false; bts.utils.BTSNativeMethods.provide_command_line(new string[] { }, 0); mbts.config.Configuration.initialize(); if (run_echo_tcp_server_only) {