From 90ec64d25c1bd579ca773e7ef5d7c92c9c61b8d8 Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Fri, 17 May 2024 12:16:35 -0400 Subject: [PATCH] deleting old serializers --- .../Primitives/FileStreams/PersistorStream.cs | 10 +- .../Primitives/FileStreams/RecorderStream.cs | 120 ++--- .../Primitives/FileStreams/SnapshotStream.cs | 226 ++++----- .../DataStream/Primitives/Misc/TestStream.cs | 231 +++++++-- .../BlueData/BlueDataUtility.cs | 38 +- .../BlueData/IBlueTableIndex.cs | 454 +++++++++--------- .../Accumulation/TableAccumulator.cs | 285 ----------- .../ClientDeserializationController.cs | 67 --- .../ClientReceive/ClientDeserializer.cs | 124 ----- .../ClientReorderingRowDeserializer.cs | 34 -- .../ClientUniversalDeserializer.cs | 35 -- .../ClientUniversalRowDeserializer.cs | 69 --- .../ClientReceive/IClientDeserializer.cs | 149 ------ .../ClientReceive/IClientRowDeserializer.cs | 12 - .../PrimitiveDeserializationHelper.cs | 79 --- .../DynamicLoading/CorePrimitiveManager.cs | 155 ------ .../Misc/SerializationUtilityMethods.cs | 339 ------------- .../Messaging}/Publish/PubTableBroker.cs | 0 .../BlueDataSerializationController.cs | 57 --- .../Serialization/BlueDataSerializer.cs | 111 ----- .../Serialization/IBlueDataSerializer.cs | 21 - .../Serialization/IBlueRowSerializer.cs | 23 - .../PrimitiveSerializationHelper.cs | 96 ---- .../Serialization/UniversalRowSerializer.cs | 120 ----- .../Serialization/UniversalSerializer.cs | 16 - .../Deserialization/IPubsubRowDeserializer.cs | 25 - .../Deserialization/PubsubDeserializer.cs | 168 ------- .../PubsubUniversalDeserializer.cs | 23 - .../PubsubUniversalRowDeserializer.cs | 34 -- .../PubsubReceive/PubsubServerTagHandler.cs | 46 -- .../PubsubReceive/StatisticsLogger.cs | 102 ---- .../Messaging/PubsubReceive/TagStateBank.cs | 32 -- .../CompressionClientDeserializer.cs | 146 ------ .../CompressionPubsubDeserializer.cs | 192 -------- .../Compression/CompressionSerializer.cs | 90 ---- .../Client/BBOOptionDeserializer.cs | 11 - .../Client/ClientFixedDeserializer.cs | 49 -- .../Client/ClientFixedRowDeserializer.cs | 38 -- .../MarketOptionTheoClientDeserializer.cs | 11 - .../Publish/BBOOptionSerializer.cs | 13 - .../FixedLength/Publish/FixedRowSerializer.cs | 35 -- .../FixedLength/Publish/FixedSerializer.cs | 20 - .../Publish/MarketOptionTheoSerializer.cs | 14 - .../Pubsub/BBOOptionPubsubDeserializer.cs | 11 - .../MarketOptionTheoPubsubDeserializer.cs | 12 - .../Pubsub/PubsubFixedDeserializer.cs | 32 -- .../Pubsub/PubsubFixedRowDeserializer.cs | 37 -- .../Client/ClientFixedChangingDeserializer.cs | 56 --- .../ClientFixedChangingRowDeserializer.cs | 67 --- .../Publish/FixedChangingRowSerializer.cs | 90 ---- .../Publish/FixedChangingSerializer.cs | 110 ----- .../Pubsub/PubsubFixedChangingDeserializer.cs | 41 -- .../PubsubFixedChangingRowDeserializer.cs | 84 ---- .../LookupTableClientDeserializer.cs | 171 ------- .../LookupTableClientlRowDeserializer.cs | 107 ----- .../LookupTable/LookupTable.cs | 194 -------- .../LookupTablePubSubDeserializer.cs | 226 --------- .../Serialization/LookupTableRowSerializer.cs | 211 -------- .../Serialization/LookupTableSerializer.cs | 279 ----------- .../ClientTheoDeserializer.cs | 69 --- .../PubsubTheoDeserializer.cs | 69 --- .../Theos/Serialization/TheoSerializer.cs | 98 ---- .../AccumulatorFactory.cs | 0 .../ITableAccumulator.cs | 0 .../Subscription/TableAccumulator.cs | 285 +++++++++++ .../TableAccumulator2.cs | 0 .../Test/CompressionSerializationTest.cs | 236 --------- .../Test/FixedChangingRowSerializationTest.cs | 100 ---- .../Test/FixedChangingSerializationTest.cs | 40 -- .../Test/MarketOptionTheoSerializationTest.cs | 73 --- .../Messaging/Test/Performance.cs | 46 -- .../Messaging/Test/TableAccumulatorTest.cs | 275 ----------- .../Test/UniversalRowSerializerTest.cs | 58 --- .../Serialization/StreamSerializationUtils.cs | 24 + 74 files changed, 941 insertions(+), 6080 deletions(-) delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializationController.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientReorderingRowDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalRowDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientRowDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/PrimitiveDeserializationHelper.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/DynamicLoading/CorePrimitiveManager.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Misc/SerializationUtilityMethods.cs rename src/DataStreamSuite/{DataStream => DataStreamCore/Messaging}/Publish/PubTableBroker.cs (100%) delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializationController.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueDataSerializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueRowSerializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/PrimitiveSerializationHelper.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalRowSerializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalSerializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/IPubsubRowDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalRowDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/PubsubServerTagHandler.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/StatisticsLogger.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/TagStateBank.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionClientDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionPubsubDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionSerializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/BBOOptionDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/ClientFixedDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/ClientFixedRowDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/MarketOptionTheoClientDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/BBOOptionSerializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/FixedRowSerializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/FixedSerializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/MarketOptionTheoSerializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/BBOOptionPubsubDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/MarketOptionTheoPubsubDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/PubsubFixedDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/PubsubFixedRowDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Client/ClientFixedChangingDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Client/ClientFixedChangingRowDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Publish/FixedChangingRowSerializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Publish/FixedChangingSerializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Pubsub/PubsubFixedChangingDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Pubsub/PubsubFixedChangingRowDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientlRowDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/LookupTable.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/PubSubDeserialization/LookupTablePubSubDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableRowSerializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableSerializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/ClientDeserialization/ClientTheoDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/PubsubDeserialization/PubsubTheoDeserializer.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/Serialization/TheoSerializer.cs rename src/DataStreamSuite/DataStreamCore/Messaging/{Accumulation => Subscription}/AccumulatorFactory.cs (100%) rename src/DataStreamSuite/DataStreamCore/Messaging/{Accumulation => Subscription}/ITableAccumulator.cs (100%) create mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator.cs rename src/DataStreamSuite/DataStreamCore/Messaging/{Accumulation => Subscription}/TableAccumulator2.cs (100%) delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Test/CompressionSerializationTest.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Test/FixedChangingRowSerializationTest.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Test/FixedChangingSerializationTest.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Test/MarketOptionTheoSerializationTest.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Test/Performance.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Test/TableAccumulatorTest.cs delete mode 100644 src/DataStreamSuite/DataStreamCore/Messaging/Test/UniversalRowSerializerTest.cs diff --git a/src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs b/src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs index 39b01757d..fbb1022ff 100644 --- a/src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs @@ -5,6 +5,7 @@ using System.Text; using System.IO; using Google.Protobuf; using System.Globalization; +using bts; namespace BCG { // Writes all the udpates to a stream in a file. @@ -24,7 +25,7 @@ namespace BCG { private ViewStream _source_stream; private string _token; private FileStream _out_fs; - private IBlueDataSerializer _serializer; + private IStreamIOMsgSerializer _serializer;// = new UniversalStreamIOSerializer(); private string _file_name = String.Empty; private long _rollover_size; private readonly string _directory_path; @@ -94,13 +95,12 @@ namespace BCG { DataPacket dp = new_value as DataPacket; byte[] serialized_update = null; bool _write_create = _serializer == null || _n_updates_in_current_file == 0; - _serializer = _serializer ?? new UniversalSerializer(dp.schema); + _serializer = _serializer ?? new UniversalStreamIOSerializer(); if (_write_create) { // new files always starts with create - _serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update); + serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows))); } else - _serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update); - + serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows))); BinaryWriter bw = new BinaryWriter(_out_fs); long utc_epoch = epoch_now(); diff --git a/src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs b/src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs index 14993d6de..1e7da4c49 100644 --- a/src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs @@ -1,68 +1,68 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.IO; +// using System; +// using System.Collections.Generic; +// using System.Linq; +// using System.Text; +// using System.IO; -namespace BCG -{ - public class RecorderStream : ViewStream - { - private ViewStream mySource; - private DateTime myStartTime; - private FileStream myOutFile; - private string myFileName; - private IBlueDataSerializer mySerializer; - - public RecorderStream(ViewStream src, string recordFile) - { - mySource = src; - myFileName = recordFile; +// namespace BCG +// { +// public class RecorderStream : ViewStream +// { +// private ViewStream mySource; +// private DateTime myStartTime; +// private FileStream myOutFile; +// private string myFileName; +// private IBlueDataSerializer mySerializer; - myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); - } +// public RecorderStream(ViewStream src, string recordFile) +// { +// mySource = src; +// myFileName = recordFile; - protected override void myOnFirstSubscriber() - { - myStartTime = DateTime.Now; - mySource.onChange += new ChangeDelegate(mySource_onChange); - myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate); - } - - protected override void myOnLastSubscriber() - { - mySource.onChange -= mySource_onChange; - myOutFile.Close(); - } +// myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); +// } - void mySource_onChange(object newValue) - { - DataPacket dp = (DataPacket)newValue; - - BlueTable toWrite = new BlueTable(dp.schema); +// protected override void myOnFirstSubscriber() +// { +// myStartTime = DateTime.Now; +// mySource.onChange += new ChangeDelegate(mySource_onChange); +// myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate); +// } - if (mySerializer == null) - { - mySerializer = new UniversalSerializer(dp.schema); - } - foreach (IBlueRow r in dp.newRows) - toWrite.addRow(r); - - byte[] result; +// protected override void myOnLastSubscriber() +// { +// mySource.onChange -= mySource_onChange; +// myOutFile.Close(); +// } - if (myLastValue == null) - mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result); - else - mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result); +// void mySource_onChange(object newValue) +// { +// DataPacket dp = (DataPacket)newValue; - BinaryWriter bw = new BinaryWriter(myOutFile); - //write down what time this happened - bw.Write((DateTime.Now - myStartTime).TotalSeconds); - bw.Write(result.Length); - bw.Write(result); - myOutFile.Flush(); +// BlueTable toWrite = new BlueTable(dp.schema); - myFireEvent(dp); - } - } -} +// if (mySerializer == null) +// { +// mySerializer = new UniversalSerializer(dp.schema); +// } +// foreach (IBlueRow r in dp.newRows) +// toWrite.addRow(r); + +// byte[] result; + +// if (myLastValue == null) +// mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result); +// else +// mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result); + +// BinaryWriter bw = new BinaryWriter(myOutFile); +// //write down what time this happened +// bw.Write((DateTime.Now - myStartTime).TotalSeconds); +// bw.Write(result.Length); +// bw.Write(result); +// myOutFile.Flush(); + +// myFireEvent(dp); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs b/src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs index b87b3a5e9..131ab929b 100644 --- a/src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs @@ -1,12 +1,12 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.IO; -using System.Runtime.Serialization.Formatters.Binary; -using BCG.Utilities.General; -using bts.collections; -using System.Reflection.Metadata; +// using System; +// using System.Collections.Generic; +// using System.Linq; +// using System.Text; +// using System.IO; +// using System.Runtime.Serialization.Formatters.Binary; +// using BCG.Utilities.General; +// using bts.collections; +// using System.Reflection.Metadata; namespace BCG { @@ -15,115 +15,115 @@ namespace BCG { FirstOfDay, LastOfDay }; - private BlueTable _current_table; - private IClientDeserializer _deserilializer = new ClientUniversalDeserializer(); - public DateTime _snapshot_time; - public string _snapshot_file; - public IEnumerable _packets; - public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) { - DateTime dt; - var parsed = DateTime.TryParse(snapshot_date, out dt); - dt = dt.Date; - bts.Log.assert(parsed); - SpecificSnapshot specific_snapshot_enum; - parsed = Enum.TryParse(specific_snapshot, out specific_snapshot_enum); - bts.Log.assert(parsed); - string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum); - ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token); - mySetRealSource(rs); - } +// private BlueTable _current_table; +// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer(); +// public DateTime _snapshot_time; +// public string _snapshot_file; +// public IEnumerable _packets; +// public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) { +// DateTime dt; +// var parsed = DateTime.TryParse(snapshot_date, out dt); +// dt = dt.Date; +// bts.Log.assert(parsed); +// SpecificSnapshot specific_snapshot_enum; +// parsed = Enum.TryParse(specific_snapshot, out specific_snapshot_enum); +// bts.Log.assert(parsed); +// string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum); +// ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token); +// mySetRealSource(rs); +// } - public SnapshotStream(string snapshot_dt, string directory_name, string token) { - _snapshot_time = PersistorStream.parse_datetime(snapshot_dt); - string token_directory_name = $"{directory_name}\\{token}"; - _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time); - } +// public SnapshotStream(string snapshot_dt, string directory_name, string token) { +// _snapshot_time = PersistorStream.parse_datetime(snapshot_dt); +// string token_directory_name = $"{directory_name}\\{token}"; +// _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time); +// } - protected override void myOnFirstSubscriber() { - FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read); - _packets = StreamPersistorUtils.load_packets_from_file(fs); - PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time); - if (snapshot_packet != null) { - foreach (PlaybackPacket p in _packets) { - byte[] ops; - Array row_array; - _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array); - for (int i = 0; i < ops.Length; i++) { - IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; - byte op = ops[i]; - if (op == StreamOpCode.create) { - IBlueRowCollection full_table = rows_args[0]; - _current_table = new BlueTable(full_table); - } else if (op == StreamOpCode.multipleRemoveAdd) { - //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; - IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; - IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; - BlueDataUtility.absorbChanges(_current_table, dels, adds); - } else { - bts.Log.assert(false, $"Oprator: {op} is not expected"); - } - if (p.idx == snapshot_packet.idx) { - myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema))); - break; - } - } - } - } - } +// protected override void myOnFirstSubscriber() { +// FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read); +// _packets = StreamPersistorUtils.load_packets_from_file(fs); +// PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time); +// if (snapshot_packet != null) { +// foreach (PlaybackPacket p in _packets) { +// byte[] ops; +// Array row_array; +// _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array); +// for (int i = 0; i < ops.Length; i++) { +// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; +// byte op = ops[i]; +// if (op == StreamOpCode.create) { +// IBlueRowCollection full_table = rows_args[0]; +// _current_table = new BlueTable(full_table); +// } else if (op == StreamOpCode.multipleRemoveAdd) { +// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; +// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; +// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; +// BlueDataUtility.absorbChanges(_current_table, dels, adds); +// } else { +// bts.Log.assert(false, $"Oprator: {op} is not expected"); +// } +// if (p.idx == snapshot_packet.idx) { +// myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema))); +// break; +// } +// } +// } +// } +// } - private void _absorb_packet(byte[] ops, Array row_array) { - for (int i = 0; i < ops.Length; i++) { - IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; - byte op = ops[i]; - if (op == StreamOpCode.create) { - IBlueRowCollection full_table = rows_args[0]; - _current_table = new BlueTable(full_table); - } else if (op == StreamOpCode.multipleRemoveAdd) { - //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; - IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; - IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; - BlueDataUtility.absorbChanges(_current_table, dels, adds); - } else { - bts.Log.assert(false, $"Operator: {op} is not expected"); - } - } - } +// private void _absorb_packet(byte[] ops, Array row_array) { +// for (int i = 0; i < ops.Length; i++) { +// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; +// byte op = ops[i]; +// if (op == StreamOpCode.create) { +// IBlueRowCollection full_table = rows_args[0]; +// _current_table = new BlueTable(full_table); +// } else if (op == StreamOpCode.multipleRemoveAdd) { +// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; +// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; +// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; +// BlueDataUtility.absorbChanges(_current_table, dels, adds); +// } else { +// bts.Log.assert(false, $"Operator: {op} is not expected"); +// } +// } +// } - protected override void myOnLastSubscriber() { - } +// protected override void myOnLastSubscriber() { +// } - // public event SnapshotStream.Progress onProgress; - // public event Action onFileExhausted; - private PlaybackPacket _locate_packet_to_replay() { - foreach (var p in _packets) { - // returns the first non replayed packet - if (!p.replayed) - return p; - } - return null; - } +// // public event SnapshotStream.Progress onProgress; +// // public event Action onFileExhausted; +// private PlaybackPacket _locate_packet_to_replay() { +// foreach (var p in _packets) { +// // returns the first non replayed packet +// if (!p.replayed) +// return p; +// } +// return null; +// } - private void _fire_packet(byte[] ops, Array row_array) { - for (int i = 0; i < ops.Length; i++) { - IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; - byte op = ops[i]; - if (op == StreamOpCode.create) { - IBlueRowCollection full_table = rows_args[0]; - _current_table = new BlueTable(full_table); - //_current_table.addRows(full_table); - myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) : - (myLastValue as DataPacket).newRows)); - } else if (op == StreamOpCode.multipleRemoveAdd) { - //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; - IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; - IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; - BlueDataUtility.absorbChanges(_current_table, dels, adds); - myFireEvent(new DataPacket(_current_table, adds, dels)); - } else { - bts.Log.assert(false, $"Oprator: {op} is not expected"); - } +// private void _fire_packet(byte[] ops, Array row_array) { +// for (int i = 0; i < ops.Length; i++) { +// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; +// byte op = ops[i]; +// if (op == StreamOpCode.create) { +// IBlueRowCollection full_table = rows_args[0]; +// _current_table = new BlueTable(full_table); +// //_current_table.addRows(full_table); +// myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) : +// (myLastValue as DataPacket).newRows)); +// } else if (op == StreamOpCode.multipleRemoveAdd) { +// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; +// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; +// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; +// BlueDataUtility.absorbChanges(_current_table, dels, adds); +// myFireEvent(new DataPacket(_current_table, adds, dels)); +// } else { +// bts.Log.assert(false, $"Oprator: {op} is not expected"); +// } - } - } +// } +// } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs index 4f9c258f8..acae94eaa 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs @@ -50,7 +50,7 @@ namespace BCG { Tuple.Create("result", TypeCode.Boolean), Tuple.Create("exception", TypeCode.String) ); - // resgiter tests here + // register tests here // _tests.Add(new Func(_run_sub_msg_test)); // _tests.Add(new Func(_run_pub_msg_test)); // _tests.Add(new Func(_run_pubsub_server_msg_test)); @@ -58,13 +58,14 @@ namespace BCG { // _tests.Add(new Func(_run_columninfo_serialization_test)); // _tests.Add(new Func(_run_schema_serialization_test)); // _tests.Add(new Func(_row_serialization_tests_no_dbnulls)); - // _tests.Add(new Func(_row_serialization_tests_with_dbnulls)); - // _tests.Add(new Func(_row_serialization_tests_all_types)); + // _tests.Add(new Func(_row_serialization_tests_with_dbnulls)); + //_tests.Add(new Func(_row_serialization_tests_all_types)); // _tests.Add(new Func(_run_full_stream_serialization_test)); // _tests.Add(new Func(_run_full_stream_serialization_with_controller_test)); // _tests.Add(new Func(_run_pubsub_server_stream_tag_handler_test)); - // _tests.Add(new Func(_run_single_tag_data_test_clean)); - _tests.Add(new Func(_run_datetime_seriliaztion_test)); + // _tests.Add(new Func(_run_single_tag_data_test_clean)); + _tests.Add(new Func(_cpp_data_test)); + _tests.Add(new Func(_write_updates_as_files)); } protected override void myOnFirstSubscriber() { @@ -200,21 +201,15 @@ namespace BCG { }; } + private TestResult _run_schema_serialization_test() { - var table_name = "test_table";//Guid.NewGuid().ToString(); - BlueTableSchema schema = new BlueTableSchema(); - List cis = new List(); - foreach (Type t in BlueDataUtility.valid_column_types()) { - if (t == null) - continue; - var col_name = t.ToString() + "_" + cis.Count.ToString(); // Guid.NewGuid().ToString(); - ColumnInfo ci = new ColumnInfo(col_name, cis.Count, t); - cis.Add(ci); - } - schema = new BlueTableSchema(table_name, cis.ToArray()); + var table_name = "test_table"; + var excluded_types = new HashSet() { typeof(DateTime), typeof(Decimal) }; + IBlueTableSchema schema = create_schema_with_all_types(excluded_types); MemoryStream str = new MemoryStream(); BinaryWriter bw = new BinaryWriter(str); StreamSerializationUtils.serialize_schema(schema, ref bw); + _write_to_file(str, @"c:\Temp\tests\schema.bin"); var hex_txt = StreamSerializationUtils.to_hex(str); bts.Log.Gen.info($"_run_schema_serialization_test ser: {hex_txt}"); @@ -230,6 +225,8 @@ namespace BCG { } + + private TestResult _run_datetime_seriliaztion_test() { DateTime dt = DateTime.Now; DateTimeOffset dt_off = new DateTimeOffset(dt); @@ -329,20 +326,29 @@ namespace BCG { } private TestResult _row_serialization_tests_no_dbnulls() { - IBlueTableSchema schema = BlueDataUtility.form_schema("test_table", - new Tuple("id", TypeCode.Int32), - new Tuple("i1", TypeCode.Int32), - new Tuple("d1", TypeCode.Double), - new Tuple("string_1", TypeCode.String)); + // IBlueTableSchema schema = BlueDataUtility.form_schema("test_table", + // new Tuple("id", TypeCode.Int32), + // new Tuple("i1", TypeCode.Int32), + // new Tuple("d1", TypeCode.Double), + // new Tuple("string_1", TypeCode.String)); + // bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema); + // IBlueRow r = new BlueRow(schema, 0, 100, 123.45, "test_string"); + var schema = create_schema_with_all_types(new HashSet() { typeof(DateTime), typeof(Decimal) }); bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema); - IBlueRow r = new BlueRow(schema, 0, 100, 123.45, "test_string"); MemoryStream str = new MemoryStream(); + IBlueRow r = generate_random_row(schema, 0, false); BinaryWriter bw = new BinaryWriter(str); row_ser.serialize_row(r, 0, ref bw); + _write_to_file(str, @"C:\Temp\tests\row.bin"); str.Position = 0; + byte[] raw_byte_array = str.ToArray(); + var hex_txt = StreamSerializationUtils.to_hex(raw_byte_array); + Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls Row bytes: {hex_txt}"); BinaryReader br = new BinaryReader(str); bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema); IBlueRow r_prime = des.deserialize_row(ref br); + string row_str = r_prime.to_string(); + Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls r_prime: {row_str}"); var passes = BlueDataUtility.compute_equality(r, r_prime); return new TestResult { test_name = "UniversalRowSerialization_NonNull", @@ -350,6 +356,156 @@ namespace BCG { }; } + private TestResult _cpp_data_test() { + //create and write schema + IBlueTableSchema schema = BlueDataUtility.form_schema("test_table", + new Tuple("id", TypeCode.Int32), + new Tuple("i_1", TypeCode.Int32), + new Tuple("i_2", TypeCode.Int32), + new Tuple("d_1", TypeCode.Double), + new Tuple("string_1", TypeCode.String) + ); + bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema); + UniversalRowDeserializer row_deser = new UniversalRowDeserializer(schema); + StreamIOSerializationController ser_controller = new StreamIOSerializationController(); + StreamIODeserializationController deser_controller = new StreamIODeserializationController(); + + IBlueTableSchema schema_p = null; + var schema_file = @"c:\Temp\tests\schema.bin"; + var row_file = @"C:\Temp\tests\row.bin"; + var create_op_file = @"C:\Temp\tests\create_op.bin"; + var mra_op_file = @"C:\Temp\tests\mra_op.bin"; + IBlueRow row = new BlueRow(schema, 0, 100, 200, 123.45, "test_string"); + + //serialize schema, write it to a file, read it back, deserialize it and match schemas + byte[] ser_schema_bytes = StreamSerializationUtils.serialize_schema(schema); + _write_to_file(ser_schema_bytes, schema_file); + + byte[] schema_file_bytes = _read_from_file(schema_file); + schema_p = StreamSerializationUtils.deserialize_schema(schema_file_bytes); + bool passes = BlueDataUtility.compute_equality(schema, schema_p); + + //serialize row, write it to a file, read it back, deserialize it and match schemas + byte[] ser_row_bytes = row_ser.serialize_row(row); + _write_to_file(ser_row_bytes, row_file); + + byte[] row_file_bytes = _read_from_file(row_file); + var row_file_bytes_txt = StreamSerializationUtils.to_hex(row_file_bytes); + Log.Gen.info($"TestsStream._cpp_data_test row_file_bytes_txt: {row_file_bytes_txt}"); + + IBlueRow row_p = row_deser.deserialize_row(row_file_bytes); + passes = passes && BlueDataUtility.compute_equality(row, row_p); + + { + // serialize a create operation + int n_rows = 100; + BlueTable table = new BlueTable(schema); + for (int i = 0; i < n_rows; i++) { + IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string"); + table.addRow(nr); + } + StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table)); + //StreamIOSerializationController ser_controller = new StreamIOSerializationController(); + byte[] create_ser_bytes = ser_controller.serialize(cr_msg); + _write_to_file(create_ser_bytes, create_op_file); + + // read it and make sure its right + byte[] create_op_file_bytes = _read_from_file(create_op_file); + //StreamIODeserializationController deser_controller = new StreamIODeserializationController(); + var cr_msg_p = deser_controller.deserialize(create_op_file_bytes); + _write_to_file(create_ser_bytes, mra_op_file); + + passes = passes && cr_msg.Equals(cr_msg_p); + + } + //multiple remove adds + { + // remove all rows and add them again + int n_rows = 100; + BlueTable table = new BlueTable(schema); + for (int i = 0; i < n_rows; i++) { + IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string"); + table.addRow(nr); + } + StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(table, table)); + + //StreamIOSerializationController ser_controller = new StreamIOSerializationController(); + byte[] mra_ser_bytes = ser_controller.serialize(mra_msg); + _write_to_file(mra_ser_bytes, mra_op_file); + + // reasd it and make sure it matches + byte[] mra_op_file_bytes = _read_from_file(mra_op_file); + //StreamIODeserializationController deser_controller = new StreamIODeserializationController(); + StreamIOMessage mra_msg_p = deser_controller.deserialize(mra_op_file_bytes); + passes = passes && mra_msg.Equals(mra_msg_p); + } + return new TestResult { + test_name = "_cpp_data_test", + test_result = passes + }; + } + + private TestResult _write_updates_as_files() { + IBlueTableSchema schema = BlueDataUtility.form_schema("test_table", + new Tuple("id", TypeCode.Int32), + new Tuple("i_1", TypeCode.Int32), + new Tuple("i_2", TypeCode.Int32), + new Tuple("d_1", TypeCode.Double), + new Tuple("string_1", TypeCode.String) + ); + var update_path = @"C:\Temp\tests\update_packets\"; + Func now_epochs = () => { + DateTimeOffset now = DateTimeOffset.UtcNow; + // Convert to Unix time in milliseconds + long unix_time_milliseconds = now.ToUnixTimeMilliseconds(); + return unix_time_milliseconds; + }; + Func curr_file_path = () => { + var epoch_ms = now_epochs(); + var file_path = $"{update_path}{epoch_ms}.bin"; + return file_path; + }; + // write 1000 stream updates + { + int init_sz = 100; + BlueTable table = new BlueTable(schema); + for (int i = 0; i < init_sz; i++) { + IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string"); + table.addRow(nr); + } + int n_updates = 1000; + int period_ms = 50; + StreamIOSerializationController ser_controller = new StreamIOSerializationController(); + for (int i = 0; i < n_updates; i++) { + if (i == 0) { + // creation + StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table)); + byte[] create_ser_bytes = ser_controller.serialize(cr_msg); + var file_path = curr_file_path(); + _write_to_file(create_ser_bytes, file_path); + } else { + // addition + BlueTable adds = new BlueTable(schema); + int id = table.Count; + IBlueRow nr = new BlueRow(schema, id, 100, 200, 123.45, "test_string"); + adds.addRow(nr); + BlueTable dels = new BlueTable(schema); + StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(dels, adds)); + byte[] mra_ser_bytes = ser_controller.serialize(mra_msg); + var file_path = curr_file_path(); + _write_to_file(mra_ser_bytes, file_path); + } + System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(period_ms)); + } + // now start writing increments and add 1 row at a time + } + return new TestResult { + test_name = "_write_updates_as_files", + test_result = true + }; + } + + private TestResult _row_serialization_tests_with_dbnulls() { IBlueTableSchema schema = BlueDataUtility.form_schema("test_table", new Tuple("id", TypeCode.Int32), @@ -375,10 +531,11 @@ namespace BCG { private TestResult _row_serialization_tests_all_types() { var schema = create_schema_with_all_types(); bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema); - IBlueRow r = generate_random_row(schema, 0); + IBlueRow r = generate_random_row(schema, 0, false); MemoryStream str = new MemoryStream(); BinaryWriter bw = new BinaryWriter(str); row_ser.serialize_row(r, 0, ref bw); + _write_to_file(str, @"C:\Temp\tests\row.bin"); str.Position = 0; BinaryReader br = new BinaryReader(str); bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema); @@ -499,6 +656,25 @@ namespace BCG { #region Helper functions + private static void _write_to_file(byte[] ser_bytes, string file_name) { + _write_to_file(new MemoryStream(ser_bytes), file_name); + } + + private static void _write_to_file(MemoryStream str, string file_name) { + if (String.IsNullOrEmpty(file_name)) + return; + byte[] str_bytes = str.ToArray(); + using (FileStream fs = new FileStream(file_name, FileMode.Create)) { + // Write the byte array to the file + fs.Write(str_bytes, 0, str_bytes.Length); + } + } + + private static byte[] _read_from_file(string file_path) { + var rv = File.ReadAllBytes(file_path); + return rv; + } + public static IBlueRowCollection get_rows_from_single_tag_state(ref SingleTagData st_data, ref IStreamMsgDeserilizer deser) { var content = st_data.get_current_content(); var msg_p = deser.deserialize(content); @@ -541,12 +717,13 @@ namespace BCG { return pub_msg; } - public static IBlueTableSchema create_schema_with_all_types() { + public static IBlueTableSchema create_schema_with_all_types(HashSet excluded_col_map = null) { + excluded_col_map = excluded_col_map == null ? new HashSet() : excluded_col_map; var table_name = "test_table"; BlueTableSchema schema = new BlueTableSchema(); List cis = new List(); foreach (Type t in BlueDataUtility.valid_column_types()) { - if (t == null/* || t == typeof(DateTime)*/) + if (t == null || t == typeof(DateTime) || excluded_col_map.Contains(t)) continue; var col_name = t.ToString() + "_" + cis.Count.ToString(); // Guid.NewGuid().ToString(); ColumnInfo ci = new ColumnInfo(col_name, cis.Count, t); @@ -557,7 +734,7 @@ namespace BCG { return schema; } - public static IBlueRow generate_random_row(IBlueTableSchema schema, int id) { + public static IBlueRow generate_random_row(IBlueTableSchema schema, int id, bool include_dbnulls = true) { Random rnd = new Random(); object[] row_vals = new object[schema.columnInfo.Length]; foreach (ColumnInfo ci in schema.columnInfo) { @@ -566,7 +743,7 @@ namespace BCG { continue; } object v = get_random_object(ci.type_code()); - if (rnd.NextSingle() % 4 == 0) + if (include_dbnulls && rnd.NextSingle() % 4 == 0) v = DBNull.Value; row_vals[ci.colNo] = v; } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs index ade6dac40..cf916f45c 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs @@ -221,23 +221,23 @@ namespace BCG { // these are the only types allowed in schemas private static IReadOnlyDictionary _typecode_to_type = new Dictionary { - { TypeCode.Boolean, typeof(bool) }, - { TypeCode.Byte, typeof(byte) }, - { TypeCode.Char, typeof(char) }, + { TypeCode.Boolean, typeof(bool) }, //3 + { TypeCode.Byte, typeof(byte) }, //6 + { TypeCode.Char, typeof(char) }, //4 // TODO - { TypeCode.DateTime, typeof(DateTime) }, - { TypeCode.Decimal, typeof(decimal) }, - { TypeCode.Double, typeof(double) }, + { TypeCode.DateTime, typeof(DateTime) },//16 + { TypeCode.Decimal, typeof(decimal) }, //15 + { TypeCode.Double, typeof(double) }, //14 //{ TypeCode.Empty, null }, - { TypeCode.Int16, typeof(short) }, - { TypeCode.Int32, typeof(int) }, - { TypeCode.Int64, typeof(long) }, - { TypeCode.SByte, typeof(sbyte) }, - { TypeCode.Single, typeof(Single) }, - { TypeCode.String, typeof(string) }, - { TypeCode.UInt16, typeof(UInt16) }, - { TypeCode.UInt32, typeof(UInt32) }, - { TypeCode.UInt64, typeof(UInt64) } + { TypeCode.Int16, typeof(short) }, //7 + { TypeCode.Int32, typeof(int) }, //9 + { TypeCode.Int64, typeof(long) }, //11 + { TypeCode.SByte, typeof(sbyte) }, //5 + { TypeCode.Single, typeof(Single) }, //13 + { TypeCode.String, typeof(string) }, //18 + { TypeCode.UInt16, typeof(UInt16) }, //8 + { TypeCode.UInt32, typeof(UInt32) }, //10 + { TypeCode.UInt64, typeof(UInt64) } //12 }; public static IEnumerable valid_column_typecodes() { @@ -1139,6 +1139,14 @@ namespace BCG { return ci.colNo + ";" + ci.colName + ";" + ci.colType; } + public static string to_string(this IBlueRow r) { + string rv = String.Empty; + foreach (ColumnInfo c in r.schema.columnInfo) { + rv += $"{c.colName}: {r[c.colName]}\n"; + } + return rv; + } + public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows, out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) { if (old_rows == null || old_rows.Count == 0) { diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/IBlueTableIndex.cs b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueTableIndex.cs index de72a4e2e..276bd3145 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/IBlueTableIndex.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueTableIndex.cs @@ -1,252 +1,252 @@ -using System; -using System.Collections; -//using BCG.Utilities; +// using System; +// using System.Collections; +// //using BCG.Utilities; -namespace BCG -{ - /// - /// Provides a multiple indexing mechanism, required to increase the peformance of the streams - /// - public class BlueTableIndexException : Exception - { - public BlueTableIndexException(string msg) : base(msg) { } - } +// namespace BCG +// { +// /// +// /// Provides a multiple indexing mechanism, required to increase the peformance of the streams +// /// +// public class BlueTableIndexException : Exception +// { +// public BlueTableIndexException(string msg) : base(msg) { } +// } - public interface IBlueTableIndex - { - void addRow(IBlueRow row); - void deleteRowOnPrimaryKey(object pkValue); - void deleteRowOnPrimaryKey(IBlueRow row); - void deleteRowOnBothKeys(object pkValue, object skValue); - void deleteRowOnBothKeys(IBlueRow row); - ArrayList searchRowOnPrimaryKey(object pkValue); - ArrayList searchRowOnPrimaryKey(IBlueRow row); - ArrayList searchRowOnBothKeys(object pkValue, object skValue); - ArrayList searchRowOnBothKeys(IBlueRow row); +// public interface IBlueTableIndex +// { +// void addRow(IBlueRow row); +// void deleteRowOnPrimaryKey(object pkValue); +// void deleteRowOnPrimaryKey(IBlueRow row); +// void deleteRowOnBothKeys(object pkValue, object skValue); +// void deleteRowOnBothKeys(IBlueRow row); +// ArrayList searchRowOnPrimaryKey(object pkValue); +// ArrayList searchRowOnPrimaryKey(IBlueRow row); +// ArrayList searchRowOnBothKeys(object pkValue, object skValue); +// ArrayList searchRowOnBothKeys(IBlueRow row); - } +// } - /* - public class BlueTableDoubleIndex : IBlueTableIndex - { - private IBlueRowCollection myTable; - private string myPrimaryIndex; - private string mySecondaryIndex; - private Hashtable myHt = new Hashtable (); - - /// - /// Builds an index with 2 columns of the table acting as primary index and secondary index - /// respectively - /// - /// - /// table to index - /// - /// - /// primary key to index upon - /// - /// - /// column serving as the secondary key - /// - public BlueTableDoubleIndex (IBlueRowCollection table, string primaryIndex, string secondaryIndex) - { - myTable = table; - myPrimaryIndex = primaryIndex; - mySecondaryIndex = secondaryIndex; - myInit (); - } - - private void myInit() - { - foreach (IBlueRow row in myTable) - addRow (row); - } +// /* +// public class BlueTableDoubleIndex : IBlueTableIndex +// { +// private IBlueRowCollection myTable; +// private string myPrimaryIndex; +// private string mySecondaryIndex; +// private Hashtable myHt = new Hashtable (); - /// - /// To add a row to the index - /// - /// - /// row to be added - /// - public void addRow (IBlueRow row) - { - if (myHt[row[myPrimaryIndex]] != null) - ((Hashtable)(myHt[row[myPrimaryIndex]]))[row[mySecondaryIndex]] = row; - else - { - TimeManager.start (); - Hashtable ht = new Hashtable(); - TimeManager.stop (); - ht[row[mySecondaryIndex]] = row; - myHt[row[myPrimaryIndex]] = ht; - } - } +// /// +// /// Builds an index with 2 columns of the table acting as primary index and secondary index +// /// respectively +// /// +// /// +// /// table to index +// /// +// /// +// /// primary key to index upon +// /// +// /// +// /// column serving as the secondary key +// /// +// public BlueTableDoubleIndex (IBlueRowCollection table, string primaryIndex, string secondaryIndex) +// { +// myTable = table; +// myPrimaryIndex = primaryIndex; +// mySecondaryIndex = secondaryIndex; +// myInit (); +// } - /// - /// If index or the primary key has already been deleted appropriate exceptions are thrown - /// - /// - public void deleteRow (IBlueRow row) - { - if (myHt[row[myPrimaryIndex]] == null) - { - throw new BlueTableIndexException ("Theres no index (primary hash table key) for this index value"); - } - ((Hashtable)(myHt[row[myPrimaryIndex]])).Remove(row[mySecondaryIndex]); - if (((Hashtable)(myHt[row[myPrimaryIndex]])).Keys.Count == 0) - myHt.Remove(row[myPrimaryIndex]); - } +// private void myInit() +// { +// foreach (IBlueRow row in myTable) +// addRow (row); +// } - /// - /// Returns an array of DataRow that match the primary key being passed as an argument - /// - /// - /// Primary Key of the table - /// - /// - /// Array of rows matching the criteria - /// - public IBlueRow[] findValueFromPrimaryKey (object o) - { - object oht = myHt[o]; - if (oht == null) - return new IBlueRow[0]; - Hashtable ht = (Hashtable)oht; - IBlueRow[] rows = new IBlueRow[ht.Keys.Count]; - ht.Values.CopyTo(rows, 0); - return rows; - } +// /// +// /// To add a row to the index +// /// +// /// +// /// row to be added +// /// +// public void addRow (IBlueRow row) +// { +// if (myHt[row[myPrimaryIndex]] != null) +// ((Hashtable)(myHt[row[myPrimaryIndex]]))[row[mySecondaryIndex]] = row; +// else +// { +// TimeManager.start (); +// Hashtable ht = new Hashtable(); +// TimeManager.stop (); +// ht[row[mySecondaryIndex]] = row; +// myHt[row[myPrimaryIndex]] = ht; +// } +// } + +// /// +// /// If index or the primary key has already been deleted appropriate exceptions are thrown +// /// +// /// +// public void deleteRow (IBlueRow row) +// { +// if (myHt[row[myPrimaryIndex]] == null) +// { +// throw new BlueTableIndexException ("Theres no index (primary hash table key) for this index value"); +// } +// ((Hashtable)(myHt[row[myPrimaryIndex]])).Remove(row[mySecondaryIndex]); +// if (((Hashtable)(myHt[row[myPrimaryIndex]])).Keys.Count == 0) +// myHt.Remove(row[myPrimaryIndex]); +// } + +// /// +// /// Returns an array of DataRow that match the primary key being passed as an argument +// /// +// /// +// /// Primary Key of the table +// /// +// /// +// /// Array of rows matching the criteria +// /// +// public IBlueRow[] findValueFromPrimaryKey (object o) +// { +// object oht = myHt[o]; +// if (oht == null) +// return new IBlueRow[0]; +// Hashtable ht = (Hashtable)oht; +// IBlueRow[] rows = new IBlueRow[ht.Keys.Count]; +// ht.Values.CopyTo(rows, 0); +// return rows; +// } - /// - /// returns as array of the primary keys - /// - /// - /// object[] of primary keys - /// - public Array getPrimaryKeys() - { - object[] retval = new object [myHt.Keys.Count]; - myHt.Keys.CopyTo (retval, 0); - return retval; - } - } - */ +// /// +// /// returns as array of the primary keys +// /// +// /// +// /// object[] of primary keys +// /// +// public Array getPrimaryKeys() +// { +// object[] retval = new object [myHt.Keys.Count]; +// myHt.Keys.CopyTo (retval, 0); +// return retval; +// } +// } +// */ - /// - /// Provides a 1-Level indexing structure. - /// - public class BlueTableSingleIndex : IBlueTableIndex - { - string myPrimaryKey; - string mySecondaryKey; - Hashtable myHt = new Hashtable(); +// /// +// /// Provides a 1-Level indexing structure. +// /// +// public class BlueTableSingleIndex : IBlueTableIndex +// { +// string myPrimaryKey; +// string mySecondaryKey; +// Hashtable myHt = new Hashtable(); - public BlueTableSingleIndex(string primaryKey, string secondaryKey) - { - myPrimaryKey = primaryKey; - mySecondaryKey = secondaryKey; - } +// public BlueTableSingleIndex(string primaryKey, string secondaryKey) +// { +// myPrimaryKey = primaryKey; +// mySecondaryKey = secondaryKey; +// } - /// - /// The primary key is the indexing key, and the secondary key is used as the key for - /// searching and hence deletion - /// - /// - /// - /// - public BlueTableSingleIndex(IBlueRowCollection table, string primaryKey, string secondaryKey) - : this(primaryKey, secondaryKey) - { - foreach (IBlueRow row in table) - addRow(row); - } +// /// +// /// The primary key is the indexing key, and the secondary key is used as the key for +// /// searching and hence deletion +// /// +// /// +// /// +// /// +// public BlueTableSingleIndex(IBlueRowCollection table, string primaryKey, string secondaryKey) +// : this(primaryKey, secondaryKey) +// { +// foreach (IBlueRow row in table) +// addRow(row); +// } - /// - /// Adds row to the indexing new structure - /// - /// - public void addRow(IBlueRow row) - { - if (myHt[row[myPrimaryKey]] == null) - myHt[row[myPrimaryKey]] = new ArrayList(); - ((ArrayList)(myHt[row[myPrimaryKey]])).Add(row); - } +// /// +// /// Adds row to the indexing new structure +// /// +// /// +// public void addRow(IBlueRow row) +// { +// if (myHt[row[myPrimaryKey]] == null) +// myHt[row[myPrimaryKey]] = new ArrayList(); +// ((ArrayList)(myHt[row[myPrimaryKey]])).Add(row); +// } - /// - /// Getts rows based on the Primary Key. Matching of the primary key is the only criteria - /// - /// - /// - public ArrayList searchRowOnPrimaryKey(IBlueRow row) - { - return searchRowOnPrimaryKey(row[myPrimaryKey]); - } +// /// +// /// Getts rows based on the Primary Key. Matching of the primary key is the only criteria +// /// +// /// +// /// +// public ArrayList searchRowOnPrimaryKey(IBlueRow row) +// { +// return searchRowOnPrimaryKey(row[myPrimaryKey]); +// } - public ArrayList searchRowOnPrimaryKey(object pkValue) - { - if (myHt[pkValue] == null) - return new ArrayList(); - return (ArrayList)myHt[pkValue]; - } +// public ArrayList searchRowOnPrimaryKey(object pkValue) +// { +// if (myHt[pkValue] == null) +// return new ArrayList(); +// return (ArrayList)myHt[pkValue]; +// } - /// - /// Gets rows based on the primary AND the secondary key - /// - /// - /// - public ArrayList searchRowOnBothKeys(IBlueRow row) - { - return searchRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]); - } +// /// +// /// Gets rows based on the primary AND the secondary key +// /// +// /// +// /// +// public ArrayList searchRowOnBothKeys(IBlueRow row) +// { +// return searchRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]); +// } - public ArrayList searchRowOnBothKeys(object pkValue, object skValue) - { - if (myHt[pkValue] == null) - return new ArrayList(); - ArrayList matchingList = (ArrayList)myHt[pkValue]; - ArrayList retVal = new ArrayList(); - foreach (IBlueRow r in matchingList) - if (r[mySecondaryKey].Equals(skValue)) - retVal.Add(r); - return retVal; - } +// public ArrayList searchRowOnBothKeys(object pkValue, object skValue) +// { +// if (myHt[pkValue] == null) +// return new ArrayList(); +// ArrayList matchingList = (ArrayList)myHt[pkValue]; +// ArrayList retVal = new ArrayList(); +// foreach (IBlueRow r in matchingList) +// if (r[mySecondaryKey].Equals(skValue)) +// retVal.Add(r); +// return retVal; +// } - /// - /// Deletes rows based on the primary key only - /// - /// - public void deleteRowOnPrimaryKey(IBlueRow row) - { - deleteRowOnPrimaryKey(row[myPrimaryKey]); - } +// /// +// /// Deletes rows based on the primary key only +// /// +// /// +// public void deleteRowOnPrimaryKey(IBlueRow row) +// { +// deleteRowOnPrimaryKey(row[myPrimaryKey]); +// } - public void deleteRowOnPrimaryKey(object pkValue) - { - if (myHt[pkValue] == null) - throw new BlueTableIndexException("Key to be deleted was not found in the index"); - myHt.Remove(pkValue); - } +// public void deleteRowOnPrimaryKey(object pkValue) +// { +// if (myHt[pkValue] == null) +// throw new BlueTableIndexException("Key to be deleted was not found in the index"); +// myHt.Remove(pkValue); +// } - public void deleteRowOnBothKeys(IBlueRow row) - { - deleteRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]); - } +// public void deleteRowOnBothKeys(IBlueRow row) +// { +// deleteRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]); +// } - public void deleteRowOnBothKeys(object pkValue, object skValue) - { - if (myHt[pkValue] == null) - throw new BlueTableIndexException("Key to be deleted was not found in the index"); - ArrayList matchingList = (ArrayList)myHt[pkValue]; - ArrayList aux = new ArrayList(); - foreach (IBlueRow r in matchingList) - if (r[mySecondaryKey].Equals(skValue)) - aux.Add(r); - foreach (IBlueRow r in aux) - matchingList.Remove(r); - } - } -} +// public void deleteRowOnBothKeys(object pkValue, object skValue) +// { +// if (myHt[pkValue] == null) +// throw new BlueTableIndexException("Key to be deleted was not found in the index"); +// ArrayList matchingList = (ArrayList)myHt[pkValue]; +// ArrayList aux = new ArrayList(); +// foreach (IBlueRow r in matchingList) +// if (r[mySecondaryKey].Equals(skValue)) +// aux.Add(r); +// foreach (IBlueRow r in aux) +// matchingList.Remove(r); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator.cs deleted file mode 100644 index fddc9f6f7..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator.cs +++ /dev/null @@ -1,285 +0,0 @@ -using System; -using System.Collections; -using System.IO; -using BCG.Utilities; -using BCG.Utilities.General; -//using BCG.Utilities.BlueConfiguration; -using System.Collections.Generic; -using System.Threading; -using mbts; -using System.Linq; -using mbts.spark; - -namespace BCG { - public class TableAccumulator : ITableAccumulator { - private HighPerformanceTimer _timer = new HighPerformanceTimer(); - private IClientDeserializer myDeserializer = new ClientDeserializationController(); - private BlueTable myStateTable; - private ISubscriberCore mySubscriptionCore; - private TableUpdateDelegate myOnUpdateDelegate; - private string myTag; - private bool myUseIncrementsForMultipleOps = DataStreamGatewaySettings.instance.useIncrementsForMultipleOps; - - public TableAccumulator(string tag, ISubscriberCore subCore) { - myTag = tag; - mySubscriptionCore = subCore; - } - - #region ITableAccumulator Members - - public bool is_active() { return myOnUpdateDelegate != null; } - - public event TableUpdateDelegate on_update { - add { - // BlueEventLog.logEvent ("TableSubscriber being Subscribed for tag :" + myTag, BlueEventLevel.Information); - if (myOnUpdateDelegate == null) { - //ie first subscription - // BlueEventLog.logEvent ("Subscription made to PubSub for tag : " + myTag, BlueEventLevel.Information); - mySubscriptionCore.subscribe(myTag); - } - myOnUpdateDelegate = - Delegate.Combine(myOnUpdateDelegate, value) as TableUpdateDelegate; - if (myStateTable != null) - value(myStateTable, myStateTable, new BlueTable(myStateTable.schema)); - } - remove { - // BlueEventLog.logEvent ("TableSubscriber being Unsubscribed for tag :" + myTag, BlueEventLevel.Information); - if (myOnUpdateDelegate != null) { - myOnUpdateDelegate = - Delegate.Remove(myOnUpdateDelegate, value) as TableUpdateDelegate; - if (myOnUpdateDelegate == null) { - //all subscribers gone - // BlueEventLog.logEvent ("Unubscription made to PubSub for tag : " + myTag, BlueEventLevel.Information); - mySubscriptionCore.unsubscribe(myTag); - myStateTable = null; - } - } - - } - } - - public void handle_updates(IEnumerable msgs) { - update(msgs.Select(t => new MemoryStream(t)).ToArray()); - } - - public void update(Stream[] streams) { - bts.Log.Gen.info($"TableAccumulator.update tag: {myTag} n_streams: {streams.Length} stream[0].len: {streams[0].Length - streams[0].Position}"); - //checking the condition as a subscriber might unsubscribe in a callback and there might be more than one - //data element for that tag in the ViewStreamDatabase - if (myOnUpdateDelegate != null) { - byte[] ops; - Array rows; - _timer.start(); - myDeserializer.deserialize(streams, out ops, out rows); - _timer.stop(); - // BlueEventLog.logEvent(String.Format("deserializing {0} took {1} msecs", myTag, _timer.lastIntervalTime), BlueEventLevel.Performance); - _timer.start(); - myAccumulateAndFire(ops, rows); - _timer.stop(); - // BlueEventLog.logEvent(String.Format("myAccumulateAndFire for {0} took {1} msecs", myTag, _timer.lastIntervalTime), BlueEventLevel.Performance); - } - } - - #endregion - - private void myAccumulateAndFire(byte[] ops, Array rows) { - //if a non create comes before the create, the deserializer returns 0 sized arrays - if (ops.Length == 0 || rows.Length == 0) { - //ie if one is zero sized so should be the other one - // BlueEventLog.logAssert (ops.Length + rows.Length == 0); - } else { - if (ops.Length > 1) { - int index = 0; - if (myStateTable == null) { - for (index = 0; index < ops.Length; index++) - if (ops[index] == StreamOpCode.create) - break; - if (index == ops.Length) - return; - } - if (myUseIncrementsForMultipleOps) - myHandleMultipleOpsIncrementally(ops, rows, index); - else - myHandleMultipleOps(ops, rows, index); - } else { - if (myStateTable != null || ops[0] == StreamOpCode.create) - myHandleSingleOp(ops, rows); - } - } - } - - private void myHandleMultipleOps(byte[] ops, Array rows, int offset) { - HighPerformanceTimer timer = new HighPerformanceTimer(); - timer.start(); - IBlueRowCollection stateAtStart = myStateTable == null ? null : new BlueTable(myStateTable); - for (int i = offset; i < ops.Length; i++) { - byte op = ops[i]; - IBlueRowCollection[] tableArray = rows.GetValue(i) as IBlueRowCollection[]; - if (op == StreamOpCode.add) - myStateTable.addRows(tableArray[0]); - else if (op == StreamOpCode.remove) - myStateTable.deleteRows(tableArray[0]); - else if (op == StreamOpCode.change) - myStateTable.changeRows(tableArray[0]); - else if (op == StreamOpCode.create) - myStateTable = new BlueTable(tableArray[0]); - else if (op == StreamOpCode.multipleChange) - myStateTable.changeRows(tableArray[0]); - else if (op == StreamOpCode.multipleRemoveAdd) { - myStateTable.deleteRows(tableArray[0]); - myStateTable.addRows(tableArray[1]); - } - } - IBlueRowCollection dRows, aRows; - if (stateAtStart == null) { - dRows = new BlueTable(myStateTable.schema); - aRows = myStateTable; - } else { - BlueDataUtility.computeDifferences(myStateTable, stateAtStart, out dRows, out aRows); - } - timer.stop(); - //Console.WriteLine("multiple ops for table " + myTag + " took " + timer.lastIntervalTime + " msecs"); - myOnUpdateDelegate(myStateTable, aRows, dRows); - - } - - #region handling mutliple ops in single update call, incrementally - - private void myHandleMultipleOpsIncrementally(byte[] ops, Array rows, int offset) { - BlueTable dels = null, adds = null; - if (myStateTable != null) { - dels = new BlueTable(myStateTable.schema); - adds = new BlueTable(myStateTable.schema); - } - for (int i = offset; i < ops.Length; i++) { - byte op = ops[i]; - IBlueRowCollection[] tableArray = rows.GetValue(i) as IBlueRowCollection[]; - if (op == StreamOpCode.add) { - myHandleAdd(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds); - } else if (op == StreamOpCode.remove) { - myHandleRemove(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds); - } else if (op == StreamOpCode.change) { - myHandleChange(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds); - } else if (op == StreamOpCode.create) { - myHandleCreate(tableArray[0], ref dels, ref adds); - } else if (op == StreamOpCode.multipleChange) { - myHandleMultipleChanges(tableArray[0], dels, adds); - } else if (op == StreamOpCode.multipleRemoveAdd) { - myHandleMultipleRemovesAdds(tableArray[0], tableArray[1], dels, adds); - } - } - myOnUpdateDelegate(myStateTable, adds, dels); - } - - private void myHandleAdd(IBlueRow r, BlueTable dels, BlueTable adds) { - if (dels != null) { - myStateTable.addRow(r); - adds.addRow(r); - } - } - - private void myHandleRemove(IBlueRow r, BlueTable dels, BlueTable adds) { - if (dels != null) { - myStateTable.deleteRow(r); - if (adds.containsID((int)r["id"])) - adds.deleteRow(r); - else - dels.addRow(r); - } - } - - private void myHandleChange(IBlueRow r, BlueTable dels, BlueTable adds) { - if (dels != null) { - IBlueRow er = myStateTable[(int)r["id"]]; - myHandleRemove(er, dels, adds); - myHandleAdd(r, dels, adds); - } - } - - private void myHandleCreate(IBlueRowCollection newTable, ref BlueTable dels, ref BlueTable adds) { - if (dels == null) { - // BlueEventLog.logAssert(adds == null); - dels = new BlueTable(newTable.schema); - adds = new BlueTable(newTable); - myStateTable = new BlueTable(newTable); - } else { - dels = new BlueTable(myStateTable); - myStateTable = new BlueTable(newTable); - adds = new BlueTable(newTable); - } - } - - private void myHandleMultipleRemovesAdds(IBlueRowCollection deletedRows, IBlueRowCollection addedRows, - BlueTable dels, BlueTable adds) { - if (dels != null) { - foreach (IBlueRow r in deletedRows) - myHandleRemove(r, dels, adds); - foreach (IBlueRow r in addedRows) - myHandleAdd(r, dels, adds); - } - } - - private void myHandleMultipleChanges(IBlueRowCollection changedRows, BlueTable dels, BlueTable adds) { - if (dels != null) - foreach (IBlueRow r in changedRows) - myHandleChange(r, dels, adds); - } - - #endregion - - private void myHandleSingleOp(byte[] ops, Array rows) { - byte op = ops[0]; - IBlueRowCollection[] tableArray = rows.GetValue(0) as IBlueRowCollection[]; - IBlueRowCollection dRows, aRows; - if (op == StreamOpCode.create) { - if (myStateTable == null) { - aRows = myStateTable = new BlueTable(tableArray[0]); - dRows = new BlueTable(myStateTable.schema); - myOnUpdateDelegate(myStateTable, aRows, dRows); - } else { - IBlueRowCollection newState = new BlueTable(tableArray[0]); - BlueDataUtility.computeDifferences(newState, myStateTable, out dRows, out aRows); - if (aRows.Count > 0 || dRows.Count > 0) { - myStateTable = new BlueTable(newState); - myOnUpdateDelegate(myStateTable, aRows, dRows); - } - } - } else { - if (op == StreamOpCode.add) { - myStateTable.addRows(aRows = tableArray[0]); - dRows = new BlueTable(myStateTable.schema); - } else if (op == StreamOpCode.remove) { - myStateTable.deleteRows(dRows = tableArray[0]); - aRows = new BlueTable(myStateTable.schema); - } else if (op == StreamOpCode.change) { - IBlueRow cRow = BlueDataUtility.extractRow(tableArray[0], 0); - aRows = new BlueTable(myStateTable.schema); - dRows = new BlueTable(myStateTable.schema); - (aRows as BlueTable).addRow(cRow); - (dRows as BlueTable).addRow(myStateTable[(int)cRow["id"]]); - myStateTable.changeRow(cRow); - } else if (op == StreamOpCode.multipleChange) { - if (myStateTable.Count == tableArray[0].Count) { - //all rows are changed, so we can optimize this - dRows = myStateTable; - aRows = myStateTable = new BlueTable(tableArray[0]); - } else { - dRows = new BlueTable(myStateTable.schema); - aRows = tableArray[0]; - foreach (IBlueRow r in tableArray[0]) { - (dRows as BlueTable).addRow(myStateTable[(int)r["id"]]); - myStateTable.changeRow(r); - } - } - } else if (op == StreamOpCode.multipleRemoveAdd) { - BlueDataUtility.absorbChanges(myStateTable, dRows = tableArray[0], aRows = tableArray[1]); - } else { - aRows = dRows = null; - // BlueEventLog.logAssert(false, "unknown opcode"); - } - myOnUpdateDelegate(myStateTable, aRows, dRows); - } - } - } - -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializationController.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializationController.cs deleted file mode 100644 index 07a2e435e..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializationController.cs +++ /dev/null @@ -1,67 +0,0 @@ -using System; -using System.IO; -using System.Collections; -using BCG.Utilities; - -namespace BCG { - /// - /// Controls (ie creates dynamically) an IClientDeserializer and then passes data to it. - /// - public class ClientDeserializationController : IClientDeserializer { - private IClientDeserializer myDeserializer; - private int _n_desers = 0; - - public ClientDeserializationController() { } - - #region IClientDeserializer Members - - /// - /// Deserializes bytes into operation and data. If it gets a non create before a create, both - /// the out arrays get initialized to empty arrays - /// - /// - /// - /// - public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) { - bts.Log.Gen.info($"ClientDeserializationController.deserialize len: {streams[0].Length - streams[0].Position}"); - if (myDeserializer != null) { - myDeserializer.deserialize(streams, out theOps, out theRows); - } else { - Stream totalStream = UtilityMethods.getAggregatedStream(streams); - long initialPos = totalStream.Position; - BinaryReader br = new BinaryReader(totalStream); - byte op; - try { - op = br.ReadByte(); - } catch (EndOfStreamException) { - op = Byte.MaxValue; - } - if (op == StreamOpCode.create) { - br.ReadString(); //pubsubclientdeserializer - string deserName = br.ReadString(); - myDeserializer = Activator.CreateInstance(Type.GetType(deserName)) - as IClientDeserializer; - totalStream.Position = initialPos; - myDeserializer.deserialize(new Stream[] { totalStream }, out theOps, out theRows); - } else { - // BlueEventLog.logEvent ("Received a non create before a create", BlueEventLevel.Warning); - theOps = new byte[] { }; - theRows = new object[] { }; - } - } - var msgs = StreamIOMessage.assemble_msgs(theOps, theRows); - int i = 0; - foreach (var m in msgs) { - bts.Log.Gen.info($"ClientDeserializationController n_desers: {_n_desers} msg no: {i++}: {m}"); - } - _n_desers++; - // log it here - - } - - - #endregion - - } - -} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializer.cs deleted file mode 100644 index 3998a1a95..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientDeserializer.cs +++ /dev/null @@ -1,124 +0,0 @@ -using System; -using System.Collections; -using System.IO; - -using BCG.Utilities; -//using BCG.Utilities.BlueConfiguration; - -namespace BCG -{ - /// - /// Pairs up with the UniversalSerializer to deserialize the corresponding data. Everythign anologous - /// to UniversalSerializer. - /// - public abstract class ClientDeserializer : IClientDeserializer - { - protected IBlueTableSchema mySchema; - protected IBlueTableSchema myFirstSchema; - protected IClientRowDeserializer myRowDeserializer; - - public ClientDeserializer() - { - myFirstSchema = null; - mySchema = null; - - } - - #region IClientDeserializer Members - - void BCG.IClientDeserializer.deserialize(Stream[] streams, out byte[] theOps, out Array theRows) - { - BinaryReader br = new BinaryReader(UtilityMethods.getAggregatedStream(streams)); - - // BlueTrace.WriteDeserializeStart(streams.Length, br.BaseStream.Length); - - // emit # of streams and total length - ArrayList ops = new ArrayList(), rows = new ArrayList(); - IBlueRowCollection[] result; - while (true) - { - byte op; - try - { - int value = br.BaseStream.ReadByte(); - if (value == -1) break; - op = (byte)value; - - // BlueTrace.WriteDeserializeOperation(op); - } - catch (EndOfStreamException) - { - break; - } - if (op == StreamOpCode.create) - { - - br.ReadString(); //pubsubDeserializer - br.ReadString(); //clientDeserializer - mySchema = new BlueTableSchema(); - mySchema.deserialze(br); - //keep track of the first schema you've ever seen, so you know how to reorder columns - if (myFirstSchema == null) - myFirstSchema = mySchema; - - result = new IBlueRowCollection[] { myGetRows(br, op, br.ReadInt32()) }; - myEncorporateRows(result[0], op); - ops.Add(op); - rows.Add(result); - } - else - { - if (mySchema != null) - { - if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove) - { - result = new IBlueRowCollection[] { myGetRows(br, op, 1) }; - myEncorporateRows(result[0], op); - ops.Add(op); - rows.Add(result); - } - else if (op == StreamOpCode.multipleChange) - { - result = new IBlueRowCollection[] { myGetRows(br, op, br.ReadInt32()) }; - myEncorporateRows(result[0], op); - ops.Add(op); - rows.Add(result); - } - else if (op == StreamOpCode.multipleRemoveAdd) - { - result = - new IBlueRowCollection[] { - myGetRows (br, StreamOpCode.remove, br.ReadInt32 ()), - myGetRows (br, StreamOpCode.add, br.ReadInt32 ()) - }; - - myEncorporateRows(result[0], StreamOpCode.remove); - myEncorporateRows(result[1], StreamOpCode.add); - ops.Add(op); - rows.Add(result); - } - else - { - // BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error); - } - } - else - { - // BlueEventLog.logEvent("Received a non create operation even though the schema" + - // "has never been created", BlueEventLevel.Warning); - } - } - } - theOps = ops.ToArray(typeof(byte)) as byte[]; - theRows = rows.ToArray(); - // BlueTrace.WriteDeserializeStop(rows.Count); - } - - #endregion - - protected abstract IBlueRowCollection myGetRows(BinaryReader br, byte op, int nRows); - - protected abstract void myEncorporateRows(IBlueRowCollection rows, byte op); - - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientReorderingRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientReorderingRowDeserializer.cs deleted file mode 100644 index e9bf2b6b1..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientReorderingRowDeserializer.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.IO; - -namespace BCG -{ - class ClientReorderingRowDeserializer : IClientRowDeserializer - { - private IBlueTableSchema myOriginalSchema; - private IBlueTableSchema myOutputSchema; - private ClientUniversalRowDeserializer myDeserializer; - public ClientReorderingRowDeserializer(IBlueTableSchema originalSchema, IBlueTableSchema newSchema) - { - myOriginalSchema = originalSchema; - myOutputSchema = newSchema; - myDeserializer = new ClientUniversalRowDeserializer(newSchema); - - } - - #region IClientRowDeserializer Members - - public void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) - { - IBlueRow temp = null; - myDeserializer.deserializeRow(currentState, br, op, out temp); - r = BlueDataUtility.changeSchema(temp, myOutputSchema); - - } - - #endregion - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalDeserializer.cs deleted file mode 100644 index 514d00e41..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalDeserializer.cs +++ /dev/null @@ -1,35 +0,0 @@ -using System; -using System.Collections; -using System.IO; - -namespace BCG -{ - public class ClientUniversalDeserializer : ClientDeserializer - { - public ClientUniversalDeserializer () {} - - protected override IBlueRowCollection myGetRows (BinaryReader br, byte op, int nRows) - { - BlueTable retVal = new BlueTable (mySchema); - - if (myRowDeserializer == null) - myRowDeserializer = new ClientUniversalRowDeserializer(mySchema); - else if (op == StreamOpCode.create) - myRowDeserializer = new ClientReorderingRowDeserializer(myFirstSchema, mySchema); - - for (int i = 0; i < nRows; i++) - { - IBlueRow row; - myRowDeserializer.deserializeRow (null, br, op, out row); - retVal.addRow (row); - } - return retVal; - } - - protected override void myEncorporateRows (IBlueRowCollection rows, byte op) - { - //this does not keep state so ignore calls - } - - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalRowDeserializer.cs deleted file mode 100644 index d696615ab..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/ClientUniversalRowDeserializer.cs +++ /dev/null @@ -1,69 +0,0 @@ -using System; -using System.Collections; -using System.IO; - -using BCG.Utilities; -//using BCG.Utilities.BlueConfiguration; - -namespace BCG { - /// - /// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process - /// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations) - /// - internal class ClientUniversalRowDeserializer : IClientRowDeserializer { - private IBlueTableSchema mySchema; - private int myNullMapLength = 0; - private PrimitiveDeserializationHelper myDeserializationHelper; - private BitArray myNullMap; - private int[] mySerIndexToSchemaIndexMap; - - public ClientUniversalRowDeserializer(IBlueTableSchema schema) { - myDeserializationHelper = new PrimitiveDeserializationHelper(mySchema = schema); - myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length) / 8))); - myInitReorderSchema(); - - - } - - #region Schema Reordering initialization - private void myInitReorderSchema() { - try { - int idColIndex = mySchema["id"].colNo; - int[] temp = new int[mySchema.columnInfo.Length]; - foreach (ColumnInfo ci in mySchema.columnInfo) { - temp[ci.colNo] = - ci.colNo == idColIndex ? 0 : - ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo; - } - mySerIndexToSchemaIndexMap = new int[temp.Length]; - for (int i = 0; i < temp.Length; i++) - mySerIndexToSchemaIndexMap[temp[i]] = i; - } catch (Exception ex) { - string schema_name = mySchema == null ? "NULL" : mySchema.tableName; - string err_msg = $"ClientUniversalRowDeserializer.myInitReorderSchema table: {schema_name} ex: {ex}"; - bts.Log.Gen.error(err_msg); - throw new ApplicationException(err_msg); - } - } - #endregion - - #region IClientRowDeserializer Members - - public void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) { - br.ReadInt32(); //length information for the row not needed - myNullMap = new BitArray(br.ReadBytes(myNullMapLength)); - object[] values = new object[mySchema.columnNames.Length]; - for (int i = 0; i < values.Length; i++) { - int colNo = mySerIndexToSchemaIndexMap[i]; - values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn(colNo, br); - } - - - r = new BlueRow(mySchema, values); - - } - - #endregion - } - -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientDeserializer.cs deleted file mode 100644 index a4e4c030d..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientDeserializer.cs +++ /dev/null @@ -1,149 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; - -namespace BCG { - public class StreamIOMessage { - public enum StreamOp { - Create = 0, - Add = 1, //adds a single row - Remove = 2, //removes a single row - Change = 3, //changes a single row - MultipleRemoveAdd = 4,//removes and adds multiple rows - MultipleChange = 5 //changes multiples rows - } - public class CreateMessage { - public IBlueRowCollection rows; - public override string ToString() { - string rv = "Create message "; - if (rows == null) { - rv += " rows: null"; - return rv; - } - foreach (IBlueRow r in rows) { - rv += $"id: {r["id"]} "; - } - return rv; - } - } - public class AddMessage { - public IBlueRow added_row; - } - public class RemoveMessage { - public IBlueRow removed_row; - } - public class ChangeMessage { - public IBlueRow changed_row; - } - public class MultipleRemoveAddMessage { - public IBlueRowCollection removed; - public IBlueRowCollection added; - public override string ToString() { - string rv = "MultipleRemoveAdd removed: "; - foreach (IBlueRow r in removed) { - rv += $"id: {r["id"]} "; - } - rv += "added: "; - foreach (IBlueRow r in added) { - rv += $"id: {r["id"]} "; - } - return rv; - } - } - public class MultipleChangeMessage { - public IBlueRowCollection changed; - } - - public StreamOp op; - public CreateMessage create_message = null; - public AddMessage add_message = null; - public RemoveMessage remove_message = null; - public ChangeMessage change_message = null; - public MultipleRemoveAddMessage multiple_remove_add_message = null; - public MultipleChangeMessage multiple_change_message = null; - public StreamIOMessage(byte op_, object data) { - op = (StreamIOMessage.StreamOp)(op_); - switch (op) { - case StreamIOMessage.StreamOp.Create: - IBlueRowCollection state = null; - // TODO, not sure why it can be an array and by itself - if (data is IBlueRowCollection) state = data as IBlueRowCollection; - else if (data is IBlueRowCollection[]) state = (data as IBlueRowCollection[]).ElementAt(0); - create_message = new CreateMessage() { rows = state }; - break; - case StreamIOMessage.StreamOp.Add: - bts.Log.assert(false); - break; - case StreamIOMessage.StreamOp.Remove: - bts.Log.assert(false); - break; - case StreamIOMessage.StreamOp.Change: - bts.Log.assert(false); - break; - case StreamIOMessage.StreamOp.MultipleRemoveAdd: - IBlueRowCollection[] row_args = data as IBlueRowCollection[]; - multiple_remove_add_message = new MultipleRemoveAddMessage() { removed = row_args[0], added = row_args[1] }; - break; - case StreamIOMessage.StreamOp.MultipleChange: - //IBlueRowCollection[] row_args = data as IBlueRowCollection[]; - multiple_change_message = new MultipleChangeMessage() { changed = (data as IBlueRowCollection[])[0] }; - break; - } - } - - - public override string ToString() { - switch (op) { - case StreamIOMessage.StreamOp.Create: - return create_message.ToString(); - break; - case StreamIOMessage.StreamOp.Add: - bts.Log.assert(false); - return String.Empty; - break; - case StreamIOMessage.StreamOp.Remove: - bts.Log.assert(false); - return String.Empty; - break; - case StreamIOMessage.StreamOp.Change: - bts.Log.assert(false); - return String.Empty; - break; - case StreamIOMessage.StreamOp.MultipleRemoveAdd: - return multiple_remove_add_message.ToString(); - break; - case StreamIOMessage.StreamOp.MultipleChange: - return multiple_change_message.ToString(); - break; - } - bts.Log.assert(false); - return String.Empty; - } - public static IEnumerable assemble_msgs(byte[] ops, Array op_data) { - int i = 0; - List rv = new List(); - foreach (byte op in ops) { - object data = op_data.GetValue(i++); - rv.Add(new StreamIOMessage(op, data)); - } - return rv; - } - } - - /// - /// Performs deserialization inside the client app. The client gets either the complete data set or the - /// increments from PubSub and this interface makes sense out of them from a tabular/datastream stand point. - /// - public interface IClientDeserializer { - /// - /// deserializes the data received by the client from the pubsub - /// - /// the data in byte format received from pubsub - /// the operations that the data maps onto - /// the data corresponding to the operations, is an array of IblueRowCollection[] - void deserialize(Stream[] streams, out byte[] theOps, out Array theRows); - } -} - - diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientRowDeserializer.cs deleted file mode 100644 index 8a2e22e8d..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/IClientRowDeserializer.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.IO; -using System.Collections; - -namespace BCG { - /// - /// Deserializes one row. the current state is also provided as some deserialization schemes might use it. - /// - public interface IClientRowDeserializer { - void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r); - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/PrimitiveDeserializationHelper.cs b/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/PrimitiveDeserializationHelper.cs deleted file mode 100644 index 2d94dc878..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/ClientReceive/PrimitiveDeserializationHelper.cs +++ /dev/null @@ -1,79 +0,0 @@ -using System; -using System.IO; - -namespace BCG -{ - /// - /// Deserializes primitive types. Contructs a hash based scheme based on the index, to make everythign O(1) - /// rather than searching it up and making it O(n) - /// - public class PrimitiveDeserializationHelper - { - private delegate object DeserializeDelegate (int colNo, BinaryReader br); - private DeserializeDelegate[] myDispatchTable; - private ColumnInfo[] myCis; - - public PrimitiveDeserializationHelper (IBlueTableSchema schema) - : this (schema.columnInfo) - {} - - public PrimitiveDeserializationHelper (ColumnInfo[] cis) - { - myDispatchTable = new DeserializeDelegate [(myCis = cis).Length]; - foreach (ColumnInfo ci in cis) - myDispatchTable[ci.colNo] = myGetDispatchMethod (ci.colType); - } - - public object deserializeColumn (int colNo, BinaryReader br) - { - return myDispatchTable[colNo] (colNo, br); - } - - private DeserializeDelegate myGetDispatchMethod (Type t) - { - if (t.Equals (BlueDataValidTypes.INT)) return new DeserializeDelegate (myDeserializeInt32); - else if (t.Equals (BlueDataValidTypes.DOUBLE)) return new DeserializeDelegate (myDeserializeDouble); - else if (t.Equals (BlueDataValidTypes.STRING)) return new DeserializeDelegate (myDeserializeString); - else if (t.Equals (BlueDataValidTypes.BYTEARRAY)) return new DeserializeDelegate (myDeserializeByteArray); - else if (t.Equals (BlueDataValidTypes.CHAR)) return new DeserializeDelegate (myDeserializeChar); - else if (t.Equals (BlueDataValidTypes.LONG)) return new DeserializeDelegate (myDeserializeLong); - else if (t.Equals (BlueDataValidTypes.SBYTE)) return new DeserializeDelegate (myDeserializeSByte); - else if (t.Equals (BlueDataValidTypes.BOOL)) return new DeserializeDelegate (myDeserializeBool); - else if (t.Equals (BlueDataValidTypes.BYTE)) return new DeserializeDelegate (myDeserializeByte); - else if (t.Equals (BlueDataValidTypes.DECIMAL)) return new DeserializeDelegate (myDeserializeDecimal); - else if (t.Equals (BlueDataValidTypes.UINT)) return new DeserializeDelegate (myDeserializeUInt); - else if (t.Equals (BlueDataValidTypes.ULONG)) return new DeserializeDelegate (myDeserializeULong); - else if (t.Equals (BlueDataValidTypes.DATETIME)) return new DeserializeDelegate (myDeserializeDateTime); - else if (t.Equals (BlueDataValidTypes.SINGLE)) return new DeserializeDelegate (myDeserializeSingle); - else if (SerializationUtilityMethods.isSerializedType (t)) return new DeserializeDelegate (myDeserializeISerializedType); - else throw new ApplicationException (t.FullName + " is not a supported type"); - } - - private object myDeserializeInt32 (int colNo, BinaryReader br) {return br.ReadInt32 ();} - private object myDeserializeISerializedType (int colNo, BinaryReader br) - { - ISerializedType o = Activator.CreateInstance (myCis[colNo].colType) - as ISerializedType; - o.deserialze (br); - return o; - } - private object myDeserializeDouble (int colNo, BinaryReader br) {return br.ReadDouble ();} - private object myDeserializeString (int colNo, BinaryReader br) {return br.ReadString ();} - private object myDeserializeByteArray (int colNo, BinaryReader br) {return br.ReadBytes (br.ReadInt32 ());} - private object myDeserializeChar (int colNo, BinaryReader br) {return br.ReadChar ();} - private object myDeserializeLong (int colNo, BinaryReader br) {return br.ReadInt64 ();} - private object myDeserializeSByte (int colNo, BinaryReader br) {return br.ReadSByte ();} - private object myDeserializeBool (int colNo, BinaryReader br) {return br.ReadBoolean ();} - private object myDeserializeByte (int colNo, BinaryReader br) {return br.ReadByte ();} - private object myDeserializeDecimal (int colNo, BinaryReader br) {return br.ReadDecimal ();} - private object myDeserializeUInt (int colNo, BinaryReader br) {return br.ReadUInt32 ();} - private object myDeserializeDateTime (int colNo, BinaryReader br) - { - return new DateTime(br.ReadInt64()); - //return DateTime.FromOADate (br.ReadDouble ()); - //return new DateTime(br.ReadInt64()); - } - private object myDeserializeULong (int colNo, BinaryReader br) {return br.ReadUInt64 ();} - private object myDeserializeSingle (int colNo, BinaryReader br) {return br.ReadSingle ();} - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/DynamicLoading/CorePrimitiveManager.cs b/src/DataStreamSuite/DataStreamCore/Messaging/DynamicLoading/CorePrimitiveManager.cs deleted file mode 100644 index 78cb71a2d..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/DynamicLoading/CorePrimitiveManager.cs +++ /dev/null @@ -1,155 +0,0 @@ -using System; -using System.Collections; -using System.Reflection; -//using BCG.Utilities.BlueConfiguration; -using BCG.Utilities; - -namespace BCG -{ - public class CorePrimtiveManager - { - private IDictionary myTypeLookupTable = new Hashtable (); - private static CorePrimtiveManager mySelf; - private ArrayList myAssemblyNames = new ArrayList (); - - private CorePrimtiveManager() - { - myLoadAllTypes (); - } - - private void myLoadAllTypes () - { - //we only load one type here - loadAssembly ("X:\\bcg\\blue2\\Debug\\CppToCSharpDataStreamBridge.dll"); - } - - private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) - { - foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies()) - if (args.Name.Equals(a.FullName)) - return a; - return null; - } - - /// - /// Loads all the types associated with the assembly - /// - /// - public void loadAssembly (string assemblyName) - { - if (myAssemblyNames.Contains (assemblyName)) - return; - myAssemblyNames.Add (assemblyName); - Assembly loadedAssembly = GetType ().Assembly.FullName.Equals (assemblyName) ? - GetType ().Assembly : - Assembly.LoadFrom (assemblyName); - - foreach (Type type in loadedAssembly.GetTypes()) - if (!type.IsInterface - && !type.IsAbstract - && type.IsPublic) - { - bool isVisible = true; - string userRep = type.Name; - - if (isVisible) - { - // BlueEventLog.logAssert(myTypeLookupTable [userRep] == null); - myTypeLookupTable[userRep] = type; - } - } - } - - - - - public static CorePrimtiveManager instance - { - get {return mySelf = mySelf == null ? new CorePrimtiveManager () : mySelf;} - } - - /// - /// Returns the Type name corresponding to the primitive name - /// - /// - /// - public string find (string friendlyName) - { - if (myTypeLookupTable[friendlyName] == null) - throw new ApplicationException (friendlyName + " is not the friendly name of any of the loaded types"); - return (myTypeLookupTable[friendlyName] as Type).Name; - } - - /// - /// Constructs and returns an object of type corresponding to the friendly name - /// - /// friendly primitive name - /// args can be null or an empty ICollection if the paramterless constructor - /// is the desired one - /// - public Object build (string friendlyName, ICollection args) - { - object[] constructorArgs = null; - if (args != null && args.Count > 0) - { - constructorArgs = new object [args.Count]; - args.CopyTo (constructorArgs, 0); - } - if (myTypeLookupTable[friendlyName] == null) - throw new ApplicationException (friendlyName + " is not the friendly name of any of the loaded types"); - return - constructorArgs == null ? - Activator.CreateInstance (myTypeLookupTable[friendlyName] as Type): - Activator.CreateInstance (myTypeLookupTable[friendlyName] as Type, constructorArgs); - } - - /// - /// Returns the name of all the primitives - /// - /// - public string[] getPrimitiveNames () - { - string[] retVal = new string [myTypeLookupTable.Count]; - myTypeLookupTable.Keys.CopyTo (retVal, 0); - return retVal; - } - - /// - /// Return the types of all the streams - /// - /// - public Type[] getStreamTypes () - { - Type[] retVal = new Type [myTypeLookupTable.Count]; - myTypeLookupTable.Values.CopyTo (retVal, 0); - return retVal; - } - - public ICollection getConstructorSigns (string primitiveName) - { - ArrayList retVal = new ArrayList (); - // BlueEventLog.logAssert (myTypeLookupTable[primitiveName] != null); - Type t = myTypeLookupTable[primitiveName] as Type; - foreach (ConstructorInfo ci in t.GetConstructors()) - { - string sign = ""; - sign = t.Name + " ("; - foreach (ParameterInfo pi in ci.GetParameters ()) - sign += pi.ParameterType.Name + - " " + - pi.Name + ", "; - sign = sign.Length == (t.Name + " (").Length ? sign + ")": sign.Remove (sign.Length - 2, 2) + ")"; - retVal.Add (sign); - } - return retVal; - } - - public string getAssemblyName (string primitiveName) - { - // BlueEventLog.logAssert (myTypeLookupTable[primitiveName] != null); - return (myTypeLookupTable[primitiveName] as Type).Assembly.GetName().Name; - - } - - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Misc/SerializationUtilityMethods.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Misc/SerializationUtilityMethods.cs deleted file mode 100644 index 8879a23c8..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Misc/SerializationUtilityMethods.cs +++ /dev/null @@ -1,339 +0,0 @@ -using System; -using System.IO; -using System.Collections; - -namespace BCG -{ - public class SerializationUtilityMethods - { - private static IDictionary mySerializationDispatchTable, - myDeserializationDispatchTable; - private delegate void MySerialize (object data, BinaryWriter bw); - private delegate object MyDeserialize (Type t, BinaryReader br); - - static SerializationUtilityMethods () - { - mySerializationDispatchTable = new Hashtable (); - mySerializationDispatchTable [BlueDataValidTypes.BOOL.Name] = - new MySerialize (mySerializeBool); - mySerializationDispatchTable [BlueDataValidTypes.BYTE.Name] = - new MySerialize (mySerializeByte); - mySerializationDispatchTable [BlueDataValidTypes.CHAR.Name] = - new MySerialize (mySerializeChar); - mySerializationDispatchTable [BlueDataValidTypes.DATETIME.Name] = - new MySerialize (mySerializeDateTime); - mySerializationDispatchTable [BlueDataValidTypes.DECIMAL.Name] = - new MySerialize (mySerializeDecimal); - mySerializationDispatchTable [BlueDataValidTypes.DOUBLE.Name] = - new MySerialize (mySerializeDouble); - mySerializationDispatchTable [BlueDataValidTypes.INT.Name] = - new MySerialize (mySerializeInt32); - mySerializationDispatchTable [BlueDataValidTypes.ISERIALIZEDTYPE.Name] = - new MySerialize (mySerializeISerializedType); - mySerializationDispatchTable [BlueDataValidTypes.LONG.Name] = - new MySerialize (mySerializeLong); - mySerializationDispatchTable [BlueDataValidTypes.SBYTE.Name] = - new MySerialize (mySerializeSByte); - mySerializationDispatchTable [BlueDataValidTypes.STRING.Name] = - new MySerialize (mySerializeString); - mySerializationDispatchTable [BlueDataValidTypes.BYTEARRAY.Name] = - new MySerialize (mySerializeByteArray); - mySerializationDispatchTable [BlueDataValidTypes.UINT.Name] = - new MySerialize (mySerializeUInt); - mySerializationDispatchTable [BlueDataValidTypes.ULONG.Name] = - new MySerialize (mySerializeULong); - mySerializationDispatchTable [BlueDataValidTypes.SINGLE.Name] = - new MySerialize (mySerializeSingle); - - myDeserializationDispatchTable = new Hashtable (); - myDeserializationDispatchTable [BlueDataValidTypes.BOOL.Name] = - new MyDeserialize (myDeserializeBool); - myDeserializationDispatchTable [BlueDataValidTypes.BYTE.Name] = - new MyDeserialize (myDeserializeByte); - myDeserializationDispatchTable [BlueDataValidTypes.CHAR.Name] = - new MyDeserialize (myDeserializeChar); - myDeserializationDispatchTable [BlueDataValidTypes.DATETIME.Name] = - new MyDeserialize (myDeserializeDateTime); - myDeserializationDispatchTable [BlueDataValidTypes.DECIMAL.Name] = - new MyDeserialize (myDeserializeDecimal); - myDeserializationDispatchTable [BlueDataValidTypes.DOUBLE.Name] = - new MyDeserialize (myDeserializeDouble); - myDeserializationDispatchTable [BlueDataValidTypes.INT.Name] = - new MyDeserialize (myDeserializeInt32); - myDeserializationDispatchTable [BlueDataValidTypes.ISERIALIZEDTYPE.Name] = - new MyDeserialize (myDeserializeISerializedType); - myDeserializationDispatchTable [BlueDataValidTypes.LONG.Name] = - new MyDeserialize (myDeserializeLong); - myDeserializationDispatchTable [BlueDataValidTypes.SBYTE.Name] = - new MyDeserialize (myDeserializeSByte); - myDeserializationDispatchTable [BlueDataValidTypes.STRING.Name] = - new MyDeserialize (myDeserializeString); - myDeserializationDispatchTable [BlueDataValidTypes.BYTEARRAY.Name] = - new MyDeserialize (myDeserializeByteArray); - myDeserializationDispatchTable [BlueDataValidTypes.UINT.Name] = - new MyDeserialize (myDeserializeUInt); - myDeserializationDispatchTable [BlueDataValidTypes.ULONG.Name] = - new MyDeserialize (myDeserializeULong); - myDeserializationDispatchTable [BlueDataValidTypes.SINGLE.Name] = - new MyDeserialize (myDeserializeSingle); - } - - public static int getObjectSerializedSize (object o) - { - Type t = o.GetType (); - if (t == BlueDataValidTypes.STRING) - { - int l = (o as string).Length; - if (l < 128) - return l + 1; - if (l < 16384) - return l + 2; - if (l < 2097152) - return l + 3; - if (l < Math.Pow (2, 28)) - return l + 4; - throw new ApplicationException ("String too big"); - } - else if (t == BlueDataValidTypes.BYTEARRAY) - { - return getFixedLengthTypeSize (BlueDataValidTypes.INT) + - (o as byte[]).Length; - } - else if (isSerializedType (t)) - { - return (o as ISerializedType).getSerializedSize (); - } - return getFixedLengthTypeSize (t); - } - - public static int getFixedLengthTypeSize (Type t) - { - if (t == BlueDataValidTypes.BOOL || - t == BlueDataValidTypes.BYTE || - t == BlueDataValidTypes.CHAR || - t == BlueDataValidTypes.SBYTE) - return 1; - else if (t == BlueDataValidTypes.INT || - t == BlueDataValidTypes.UINT|| - t == BlueDataValidTypes.SINGLE) - return 4; - else if (t == BlueDataValidTypes.DOUBLE || - t == BlueDataValidTypes.LONG || - t== BlueDataValidTypes.ULONG) - return 8; - else if (t == BlueDataValidTypes.DECIMAL) - return 16; - else if (t == BlueDataValidTypes.DATETIME) - //as objects of type DateTime are stored as doubles - return getFixedLengthTypeSize (typeof(long)); - else - throw new ApplicationException (t.FullName + " is not a supported type"); - - } - - public static bool isFixedLengthType (Type t) - { - return !(t.Equals (BlueDataValidTypes.STRING) || t.Equals (BlueDataValidTypes.BYTEARRAY) || - isSerializedType (t)); - } - - public static bool isSerializedType (Type t) - { - return t.GetInterface ("BCG.ISerializedType") != null; - } - - #region Serialization - public static void serializeValue (object data, BinaryWriter bw) - { - Type t = data.GetType (); - if (isSerializedType (t)) - (mySerializationDispatchTable [typeof(ISerializedType).Name] - as MySerialize)(data, bw); - else - (mySerializationDispatchTable [t.Name] as MySerialize) (data, bw); - } - public static void serializeEnum(object data, BinaryWriter bw) - { - mySerializeInt32(data, bw); - } - private static void mySerializeInt32 (object data, BinaryWriter bw) - { - bw.Write ((Int32)data); - } - - private static void mySerializeDouble (object data, BinaryWriter bw) - { - bw.Write ((double)data); - } - - private static void mySerializeString (object data, BinaryWriter bw) - { - bw.Write (data as string); - } - - private static void mySerializeByteArray (object data, BinaryWriter bw) - { - byte[] bs = data as byte[]; - bw.Write (bs.Length); - bw.Write (bs); - } - - - private static void mySerializeSingle (object data, BinaryWriter bw) - { - bw.Write ((Single)data); - } - - private static void mySerializeChar (object data, BinaryWriter bw) - { - bw.Write ((char)data); - } - - private static void mySerializeLong (object data, BinaryWriter bw) - { - bw.Write ((long)data); - } - - private static void mySerializeSByte (object data, BinaryWriter bw) - { - bw.Write ((sbyte)data); - } - - private static void mySerializeBool (object data, BinaryWriter bw) - { - bw.Write ((bool)data); - } - - private static void mySerializeByte (object data, BinaryWriter bw) - { - bw.Write ((byte)data); - } - - private static void mySerializeDecimal (object data, BinaryWriter bw) - { - bw.Write ((decimal)data); - } - - private static void mySerializeUInt (object data, BinaryWriter bw) - { - bw.Write ((uint)data); - } - - private static void mySerializeULong (object data, BinaryWriter bw) - { - bw.Write ((ulong)data); - } - - private static void mySerializeDateTime (object data, BinaryWriter bw) - { - //DateTime is written is serialiazed as OADATE - bw.Write (((DateTime)data).ToOADate()); - } - - private static void mySerializeISerializedType (object data, BinaryWriter bw) - { - (data as ISerializedType).serialize (bw); - } - #endregion - - #region Deserialization - - public static object deserializeValue(Type t, BinaryReader br) - { - return myDeserialize(t, br); - } - private static object myDeserialize (Type t, BinaryReader br) - { - return isSerializedType (t) ? - (myDeserializationDispatchTable [typeof(ISerializedType).Name] - as MyDeserialize)(t, br) - : - (myDeserializationDispatchTable [t.Name] as MyDeserialize) (t, br); - } - - public static object deserializeEnum(Type t, BinaryReader br) - { - return myDeserializeInt32(t, br); - } - - private static object myDeserializeInt32 (Type t, BinaryReader br) - { - return br.ReadInt32 (); - } - - private static object myDeserializeISerializedType (Type t, BinaryReader br) - { - ISerializedType o = Activator.CreateInstance (t) - as ISerializedType; - o.deserialze (br); - return o; - } - - private static object myDeserializeDouble (Type t, BinaryReader br) - { - return br.ReadDouble (); - } - - private static object myDeserializeString (Type t, BinaryReader br) - { - return br.ReadString (); - } - - private static object myDeserializeByteArray (Type t, BinaryReader br) - { - return br.ReadBytes (br.ReadInt32 ()); - } - - private static object myDeserializeSingle (Type t, BinaryReader br) - { - return br.ReadSingle (); - } - - private static object myDeserializeChar (Type t, BinaryReader br) - { - return br.ReadChar (); - } - - private static object myDeserializeLong (Type t, BinaryReader br) - { - return br.ReadInt64 (); - } - - private static object myDeserializeSByte (Type t, BinaryReader br) - { - return br.ReadSByte (); - } - - private static object myDeserializeBool (Type t, BinaryReader br) - { - return br.ReadBoolean (); - } - - private static object myDeserializeByte (Type t, BinaryReader br) - { - return br.ReadByte (); - } - - private static object myDeserializeDecimal (Type t, BinaryReader br) - { - return br.ReadDecimal (); - } - - private static object myDeserializeUInt (Type t, BinaryReader br) - { - return br.ReadUInt32 (); - } - - private static object myDeserializeDateTime (Type t, BinaryReader br) - { - return DateTime.FromOADate(br.ReadDouble()); - //return new DateTime(br.ReadInt64()); - } - - private static object myDeserializeULong (Type t, BinaryReader br) - { - return br.ReadUInt64 (); - } - #endregion - } -} diff --git a/src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTableBroker.cs similarity index 100% rename from src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs rename to src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTableBroker.cs diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializationController.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializationController.cs deleted file mode 100644 index 76e9e1d69..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializationController.cs +++ /dev/null @@ -1,57 +0,0 @@ -using System; - -namespace BCG { - /// - /// Deals with the publication of one tag. Gets the specific implementaion of IBlueDataSerializer according to - /// the table name, from the config, and wraps around it. The external code that will deal with user calls for - /// row operations (PubTablePublisher) will only deal with this class, hence will be agnostic of the specalized - /// serialization. - /// - public class BlueDataSerializationController : IBlueDataSerializer { - private IBlueDataSerializer mySerializer; - - public BlueDataSerializationController() { } - - #region IBlueDataSerializer Members - - /// - /// Serializes an op and data; however if a non create is called before a create, - /// the serializedData is set to an empty byte array. - /// - /// - /// - /// - public void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData) { - if (op == StreamOpCode.create) { - if (mySerializer == null) - mySerializer = myGetSerializerForTable(rows[0].schema); - - mySerializer.serialize(op, rows, out serializedData); - } else { - if (mySerializer != null) { - mySerializer.serialize(op, rows, out serializedData); - } else { - serializedData = new byte[] { }; - } - } - } - #endregion - - private IBlueDataSerializer myGetSerializerForTable(IBlueTableSchema theSchema) { - // TODOds - return new UniversalSerializer(theSchema); - string serializerName = DataStreamGatewaySettings.instance.getSerializer(theSchema.tableName); - - object[] serializerArgs = DataStreamGatewaySettings.instance.getSerializerArgs(theSchema.tableName); - - if (serializerArgs != null) - return Activator.CreateInstance(Type.GetType(serializerName), theSchema, serializerArgs) as IBlueDataSerializer; - else - return Activator.CreateInstance(Type.GetType(serializerName), theSchema) as IBlueDataSerializer; - - - - } - - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializer.cs deleted file mode 100644 index d8e0c8f53..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/BlueDataSerializer.cs +++ /dev/null @@ -1,111 +0,0 @@ -using System; -using System.Collections; -using System.IO; - -namespace BCG -{ - /// - /// Abstract base class that can potentially serve all implementations of IBlueDataSerializer. The IBlueRowSerializer - /// is used to serialize individual rows. The implementaions should, in the constructor, contruct the myRowSerializer - /// to do the specific row serializations. Every nonabtract implementation of the this, should have a contructor - /// taking in an IBlueTableSchema. - /// - public abstract class BlueDataSerializer : IBlueDataSerializer - { - protected IBlueRowSerializer myRowSerializer; - protected string myClientDeserializerName, myPubsubDeserializerName; - - public BlueDataSerializer(string pubsubDeserializerName, - string clientDeserializerName) - { - myPubsubDeserializerName = pubsubDeserializerName; - myClientDeserializerName = clientDeserializerName; - } - - - #region IBlueDataSerializer Members - /// - /// Provides the following serialization - /// - /// -add, remove, change: - /// op - /// serializedRow (depending on the implementation of myRowSerializer) - /// - /// -create - /// op - /// pubsubDeserializer name - /// clientDeserializer name - /// schema - /// # of rows - /// serializedRows - /// - /// -multipleAddRemoves - /// op - /// # of deleted rows - /// serializedRows - /// # of added rows - /// serializedRows - /// - /// -multipleChanges - /// op - /// # of changed rows - /// serializedRows - /// - /// - /// - /// - public void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData) - { - // BlueTrace.WriteSerializeStart(op, rows.Length); - - BinaryWriter bw = new BinaryWriter(new MemoryStream()); - if (op == StreamOpCode.add || op == StreamOpCode.remove || op == StreamOpCode.change) - { - bw.Write(op); - myRowSerializer.serializeRow(BlueDataUtility.extractRow(rows[0], 0), op, bw); - } - else if (op == StreamOpCode.create) - { - IBlueRowCollection createdRows = rows[0]; - bw.Write(op); - bw.Write(myPubsubDeserializerName); - bw.Write(myClientDeserializerName); - int dummyIdCol = 0; - int[] dummyIdList = new int[1]; - //the schema doesn't sort anything right now, because the C++ pub sub client depends on things being in a certain order - IBlueTableSchema sortedSchema = BlueDataUtility.getSortedSchema(createdRows.schema, out dummyIdList, out dummyIdCol); - sortedSchema.serialize(bw); - bw.Write(createdRows.Count); - foreach (IBlueRow r in createdRows) - myRowSerializer.serializeRow(r, StreamOpCode.add, bw); - - } - else if (op == StreamOpCode.multipleRemoveAdd) - { - bw.Write(op); - IBlueRowCollection delRows = rows[0], - addRows = rows[1]; - bw.Write(delRows.Count); - foreach (IBlueRow dr in delRows) - myRowSerializer.serializeRow(dr, StreamOpCode.remove, bw); - bw.Write(addRows.Count); - foreach (IBlueRow ar in addRows) - myRowSerializer.serializeRow(ar, StreamOpCode.add, bw); - } - else if (op == StreamOpCode.multipleChange) - { - bw.Write(op); - IBlueRowCollection changedRows = rows[0]; - bw.Write(changedRows.Count); - foreach (IBlueRow cr in changedRows) - myRowSerializer.serializeRow(cr, StreamOpCode.change, bw); - } - bw.BaseStream.Position = 0; - serializedData = (bw.BaseStream as MemoryStream).ToArray(); - - // BlueTrace.WriteSerializeStop(serializedData.Length); - } - #endregion - } - -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueDataSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueDataSerializer.cs deleted file mode 100644 index 125911537..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueDataSerializer.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System; -using BCG; -using System.Collections; -using System.IO; - -namespace BCG { - /// - /// All the data serialized to be send over to PubSub happens here. All it does is f(op, rows) = byte[]. - /// - public interface IBlueDataSerializer { - /// - /// f(op, rows) = byte[] - /// - /// Operation that can be add, delete, change etc - /// The data to serialize. The only case when it will be an array of more than 1 element - /// is when there are multiple deletes and adds, ie StreamOpCode.multipleRemoveAdd. - /// - /// The data serialized as bytes to be send over the wire - void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData); - } -} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueRowSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueRowSerializer.cs deleted file mode 100644 index a7bf20d22..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueRowSerializer.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System; -using System.IO; - -namespace BCG -{ - /// - /// Responsible for serilializing rows - /// The way the serialization scheme works is that the header is same - /// for all serialization algorithms, only the way the row is serialized is different. Abstracting it out and we - /// can have each specific serializer have a unique implementation of IBlueRowSerializer. - /// - public interface IBlueRowSerializer - { - /// - /// Serializes a row and writes it into the BinaryWriter. Since its only one row, the ops should be appropriate - /// for one row; the ones allowed are StreamopCode.add, remove and change only. - /// - /// - /// - void serializeRow (IBlueRow r, byte op, BinaryWriter bw); - - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/PrimitiveSerializationHelper.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/PrimitiveSerializationHelper.cs deleted file mode 100644 index f55fd8a58..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/PrimitiveSerializationHelper.cs +++ /dev/null @@ -1,96 +0,0 @@ -using System; -using System.IO; - -namespace BCG -{ - /// - /// Serializes primitive types. Contructs a hash based scheme based on the index, to make everythign O(1) - /// rather than searching it up and making it O(n), n = types supported. - /// - public class PrimitiveSerializationHelper - { - private delegate void SerializeDelegate (object data, BinaryWriter bw); - private SerializeDelegate[] myDispatchTable; - - #region constructors - public PrimitiveSerializationHelper (IBlueTableSchema schema) - : this (schema.columnInfo) {} - - public PrimitiveSerializationHelper (ColumnInfo[] cis) - { - myDispatchTable = new SerializeDelegate [cis.Length]; - foreach (ColumnInfo ci in cis) - myDispatchTable[ci.colNo] = myGetDispatchMethod (ci.colType); - } - #endregion - - #region public methods - /// - /// Serializes the column no in the row and writes it into the BinaryWriter - /// - /// - /// - /// - public void serializeColumn (IBlueRow r, int colNo, BinaryWriter bw) - { - myDispatchTable[colNo] (r[colNo], bw); - } - - /// - /// Writes the data to the BinaryWriter. The ColNo is provided to get the appropriate - /// serializer. - /// - /// - /// - /// - public void serializeColumn (object data, int colNo, BinaryWriter bw) - { - myDispatchTable[colNo] (data, bw); - } - #endregion - - #region private methods - private void mySerializeInt32 (object data, BinaryWriter bw) {bw.Write ((Int32)data);} - private void mySerializeDouble (object data, BinaryWriter bw) {bw.Write ((double)data);} - private void mySerializeString (object data, BinaryWriter bw){bw.Write (data as string);} - private void mySerializeByteArray (object data, BinaryWriter bw) - { - byte[] bs = data as byte[]; - bw.Write (bs.Length); - bw.Write (bs); - } - private void mySerializeChar (object data, BinaryWriter bw) {bw.Write ((char)data);} - private void mySerializeLong (object data, BinaryWriter bw) {bw.Write ((long)data);} - private void mySerializeSByte (object data, BinaryWriter bw) {bw.Write ((sbyte)data);} - private void mySerializeBool (object data, BinaryWriter bw) {bw.Write ((bool)data);} - private void mySerializeByte (object data, BinaryWriter bw) {bw.Write ((byte)data);} - private void mySerializeDecimal (object data, BinaryWriter bw) {bw.Write ((decimal)data);} - private void mySerializeUInt (object data, BinaryWriter bw) {bw.Write ((uint)data);} - private void mySerializeULong (object data, BinaryWriter bw){bw.Write ((ulong)data);} - private void mySerializeDateTime (object data, BinaryWriter bw) {bw.Write (((DateTime)data).Ticks);} - private void mySerializeSingle (object data, BinaryWriter bw) {bw.Write ((Single)data);} - private void mySerializeISerializedType (object data, BinaryWriter bw) {(data as ISerializedType).serialize (bw);} - - private SerializeDelegate myGetDispatchMethod (Type t) - { - if (t.Equals (BlueDataValidTypes.INT)) return new SerializeDelegate (mySerializeInt32); - else if (t.Equals (BlueDataValidTypes.DOUBLE)) return new SerializeDelegate (mySerializeDouble); - else if (t.Equals (BlueDataValidTypes.STRING)) return new SerializeDelegate (mySerializeString); - else if (t.Equals (BlueDataValidTypes.BYTEARRAY)) return new SerializeDelegate (mySerializeByteArray); - else if (t.Equals (BlueDataValidTypes.CHAR)) return new SerializeDelegate (mySerializeChar); - else if (t.Equals (BlueDataValidTypes.LONG)) return new SerializeDelegate (mySerializeLong); - else if (t.Equals (BlueDataValidTypes.SBYTE)) return new SerializeDelegate (mySerializeSByte); - else if (t.Equals (BlueDataValidTypes.BOOL)) return new SerializeDelegate (mySerializeBool); - else if (t.Equals (BlueDataValidTypes.BYTE)) return new SerializeDelegate (mySerializeByte); - else if (t.Equals (BlueDataValidTypes.DECIMAL)) return new SerializeDelegate (mySerializeDecimal); - else if (t.Equals (BlueDataValidTypes.UINT)) return new SerializeDelegate (mySerializeUInt); - else if (t.Equals (BlueDataValidTypes.ULONG)) return new SerializeDelegate (mySerializeULong); - else if (t.Equals (BlueDataValidTypes.DATETIME)) return new SerializeDelegate (mySerializeDateTime); - else if (t.Equals (BlueDataValidTypes.SINGLE)) return new SerializeDelegate (mySerializeSingle); - else if (SerializationUtilityMethods.isSerializedType (t)) return new SerializeDelegate (mySerializeISerializedType); - else throw new ApplicationException (t.FullName + " is not a supported type"); - } - #endregion - - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalRowSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalRowSerializer.cs deleted file mode 100644 index d0aa73a3b..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalRowSerializer.cs +++ /dev/null @@ -1,120 +0,0 @@ -using System; -using System.Collections; -using System.IO; -using BCG.Utilities; -using System.Collections.Generic; -namespace BCG -{ - /// - /// Helps the UniversalSerializer by serializing one row. Hence this class is agnostic to the operation - /// which is figured out by UniversalSerializer. - /// - /// it sorts the schema it gets, to make sure that the rows are always published the same way. - /// For now, the sorting operation doesn't do anything. The reason for this is that C++ clients - /// are depending upon the data arriving in a certain order. - /// - /// It performs a mapping : (row)->byte[] - /// - /// Row serialized in the following order - /// row size (nullBitMap size + actual row size) - /// null bit map - /// non null row members (with id as the first col) - /// - /// - /// - public void serializeRow(IBlueRow r, byte op, BinaryWriter bw) - { - myNullMap.SetAll(false); - for (int i = 0; i < mySchema.columnNames.Length; i++) - if (r[mySortedSchemaToOriginalSchemaMapping[i]] == DBNull.Value || r[mySortedSchemaToOriginalSchemaMapping[i]] == null) - myNullMap[i] = true; - myNullMap.CopyTo(myNullMapArray, 0); - long initPos = bw.BaseStream.Position; - bw.Write((int)0); ///place holder for size - bw.Write(myNullMapArray); - bw.Write((int)r[mySortedSchemaToOriginalSchemaMapping[myIdColumnInSortedSchema]]); - foreach (ColumnInfo ci in mySchema.columnInfo) - if (!myNullMap[ci.colNo] && ci.colNo != myIdColumnInSortedSchema) - myPrimitiveSerializationHelper.serializeColumn(r, mySortedSchemaToOriginalSchemaMapping[ci.colNo], bw); - long endPos = bw.BaseStream.Position; - bw.BaseStream.Position = initPos; - bw.Write(Convert.ToInt32(endPos - (initPos+4))); /// size does not include the meta data hence substracting 4 - bw.BaseStream.Position = endPos; - } - - public void serializeSchema(BinaryWriter bw) - { - mySchema.serialize(bw); - } - #endregion - - #region size calculations - /// - /// Sets up the fixed length (size in bytes occupied by each of the primitive type) - /// and an index for the non primitive type - /// - private void myInitSizeCalculations() - { - ArrayList temp = new ArrayList(); - myFixedSize = myNullMapArray.Length; - foreach (ColumnInfo ci in mySchema.columnInfo) - { - if (!SerializationUtilityMethods.isFixedLengthType(ci.colType)) - temp.Add(ci.colNo); - else - myFixedSize += SerializationUtilityMethods.getFixedLengthTypeSize(ci.colType); - } - myNonFixedColIndexes = temp.ToArray(typeof(int)) as int[]; - } - - /// - /// Find the size of the serialized row, assumes that nullMap has already - /// been built - /// - /// - /// - /// - private int myGetRowSizeInBytes(int fixedSize, IBlueRow r) - { - foreach (int colNo in myNonFixedColIndexes) - { - if (!myNullMap[colNo]) - fixedSize += //myGetStringSizeInBytes (r[colNo] as string); - SerializationUtilityMethods.getObjectSerializedSize(r[mySortedSchemaToOriginalSchemaMapping[colNo]]); - } - return fixedSize; - } - #endregion - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalSerializer.cs deleted file mode 100644 index 23d803ce9..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/UniversalSerializer.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System; - -namespace BCG -{ - /// - /// Uses the UniversalRowSerializer as the IBlueRowSerializer implementation. - /// - public class UniversalSerializer : BlueDataSerializer - { - public UniversalSerializer (IBlueTableSchema schema) - : base ("BCG.PubsubUniversalDeserializer, DataStreamCore", "BCG.ClientUniversalDeserializer") - { - myRowSerializer = new UniversalRowSerializer (schema); - } - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/IPubsubRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/IPubsubRowDeserializer.cs deleted file mode 100644 index 04461d55b..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/IPubsubRowDeserializer.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System; -using System.IO; -using System.Collections; - -namespace BCG -{ - /// - /// Deserializes a row. Returns the id of the row and its full content - /// - public interface IPubsubRowDeserializer - { - /// - /// Reads the data from the BinaryReader and deserializes it in the form of a row - /// - /// Some deserializers might need the current state, ie a Dictionary of rows - /// keyed off the id - /// BinaryReader starting at the row - /// The operation, as this might drive the deserializations scheme in some cases. This op - /// however is one of the basics operations, add remove or change as all other operations can be resolved into this - /// The id of the deserialized row - /// The actual row contents - void deserializeRow (IDictionary currentState, BinaryReader br, byte op, out int id, out object row); - } - -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubDeserializer.cs deleted file mode 100644 index aafa8d573..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubDeserializer.cs +++ /dev/null @@ -1,168 +0,0 @@ -using System; -using System.Collections; -using System.IO; -using BCG.Utilities; -using System.Text; - -namespace BCG -{ - /// - /// Base class for all pubsubs deserializers. - /// - public abstract class PubsubDeserializer : IPubsubDeserializer - { - // this is where the stream state is stored in pubsub? - protected IDictionary myState = new Hashtable (); - protected IBlueTableSchema mySchema; - protected string myClientDeserializerName; //for the getState method - protected IPubsubRowDeserializer myDeserializer; - - public PubsubDeserializer () {} - - #region IPubSubDeserializer Members - public virtual void deserialize (Stream[] streams, out byte[] theOps, out Array theRows) - { - BinaryReader br = new BinaryReader (UtilityMethods.getAggregatedStream (streams)); - ArrayList ops = new ArrayList (), rows = new ArrayList (); - while (true) - { - IDictionary[] result; - byte op; - try - { - int value = br.BaseStream.ReadByte(); - if (value == -1) break; - op = (byte)value; - } - catch (EndOfStreamException) - { - // TODO, logically break loop - break; - } - if (op == StreamOpCode.create) - { - myState.Clear (); - br.ReadString (); //the pubsub-deserializer name, should be the same as this's name - myClientDeserializerName = br.ReadString (); //the client-deserializer name - (mySchema = new BlueTableSchema()).deserialze (br); - result = new IDictionary[] {myGetRows (br, op, br.ReadInt32 ())}; - myEncorporatRows (result[0], op); - ops.Add (op); - rows.Add (result); - } - else - { - if (mySchema != null) - { - if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove) - { - result = new IDictionary[] {myGetRows (br, op, 1)}; - myEncorporatRows (result[0], op); - ops.Add (op); - rows.Add (result); - } - else if (op == StreamOpCode.multipleChange) - { - result = new IDictionary[] {myGetRows (br, op, br.ReadInt32 ())}; - myEncorporatRows (result[0], op); - ops.Add (op); - rows.Add (result); - } - else if (op == StreamOpCode.multipleRemoveAdd) - { - ops.Add (op); - result = new IDictionary [] { - myGetRows (br, StreamOpCode.remove, br.ReadInt32 ()), - myGetRows (br, StreamOpCode.add, br.ReadInt32 ()) - }; - myEncorporatRows (result[0], StreamOpCode.remove); - myEncorporatRows (result[1], StreamOpCode.add); - rows.Add (result); - } - else - { - // BlueEventLog.logEvent ("Illegal op-code received", BlueEventLevel.Error); - } - } - else - { - // BlueEventLog.logEvent ("Received a non create operation even though the schema" + - // "has never been created", BlueEventLevel.Warning); - } - } - } - theOps = ops.ToArray (typeof(byte)) as byte[]; - theRows = rows.ToArray (); - } - - /// - /// Sends in a create signal, so that new clients can get a create everytime and start - /// afresh from there - /// - /// - public virtual byte[] getState() - { - if (mySchema == null) - return new byte[] {}; - else - { - BinaryWriter bw = new BinaryWriter (new MemoryStream ()); - bw.Write (StreamOpCode.create); - bw.Write (GetType().AssemblyQualifiedName); - bw.Write (myClientDeserializerName); - mySchema.serialize (bw); - bw.Write (myState.Count); - foreach (byte[] row in myState.Values) - bw.Write (row); - return (bw.BaseStream as MemoryStream).ToArray (); - } - } - #endregion - - /// - /// Every row that has been deserialized passes through this method, so that it can update the state. - /// - /// - /// - protected virtual void myEncorporatRows (IDictionary rows, byte op) - { - if (op == StreamOpCode.add || op == StreamOpCode.create) - foreach (DictionaryEntry de in rows) - { - if (myState.Contains(de.Key)) { - // BlueEventLog2.instance().logEvent(String.Format("Duplicate row creation. t: {0} id:{1}", mySchema.tableName, de.Key),BlueEventLog2Level.Fatal); - StringBuilder b = new StringBuilder(); - foreach (string n in mySchema.columnNames) - b.AppendFormat(" {0}", n); - // BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()), BlueEventLog2Level.Fatal); - } - myState.Add(de.Key, de.Value); - } - else if (op == StreamOpCode.remove) - foreach (DictionaryEntry de in rows) - { - if (!myState.Contains(de.Key)) - { - // BlueEventLog2.instance().logEvent(String.Format("Unknown row deletion. t: {0} id:{1}", mySchema.tableName, de.Key), BlueEventLog2Level.Fatal); - StringBuilder b = new StringBuilder(); - foreach (string n in mySchema.columnNames) - b.AppendFormat(" {0}", n); - // BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()),BlueEventLog2Level.Fatal); - } - myState.Remove(de.Key); - } - else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange) - foreach (DictionaryEntry de in rows) - myState[de.Key] = de.Value; - } - - /// - /// Mass deserialization happens here, also since there is no knowledge of the schema - /// - /// - /// - /// - /// - protected abstract IDictionary myGetRows (BinaryReader br, byte op, int noOfRows); - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalDeserializer.cs deleted file mode 100644 index 2dc537d4e..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalDeserializer.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System; -using System.Collections; -using System.IO; - -namespace BCG { - public class PubsubUniversalDeserializer : PubsubDeserializer { - public PubsubUniversalDeserializer() { - bts.Log.Gen.info("PubsubUniversalDeserializer.cons"); - } - - protected override IDictionary myGetRows(BinaryReader br, byte op, int noOfRows) { - myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer(mySchema) : myDeserializer; - Hashtable retVal = new Hashtable(); - for (int i = 0; i < noOfRows; i++) { - int id; object row; - myDeserializer.deserializeRow(myState, br, op, out id, out row); - retVal.Add(id, row); - } - return retVal; - } - - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalRowDeserializer.cs deleted file mode 100644 index 330c91c38..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/Deserialization/PubsubUniversalRowDeserializer.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System; -using System.IO; -using System.Collections; - - -namespace BCG -{ - public class PubsubUniversalRowDeserializer : IPubsubRowDeserializer - { - private int myNullMapLength; - private IBlueTableSchema mySchema; - private int myRowOverheadSize = SerializationUtilityMethods.getFixedLengthTypeSize (typeof(int)); - - public PubsubUniversalRowDeserializer (IBlueTableSchema schema) - { - mySchema = schema; - myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8))); - } - - #region IRowDeserializer Members - - public void deserializeRow (IDictionary currentState, BinaryReader br, byte op, out int id, out object row) - { - //current state is ignored - long pos = br.BaseStream.Position; - int rowSize = br.ReadInt32 (); - br.BaseStream.Position = pos; - row = br.ReadBytes (rowSize + myRowOverheadSize); - id = BitConverter.ToInt32 (row as byte[], myNullMapLength + myRowOverheadSize); - } - - #endregion - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/PubsubServerTagHandler.cs b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/PubsubServerTagHandler.cs deleted file mode 100644 index b95e8bbc2..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/PubsubServerTagHandler.cs +++ /dev/null @@ -1,46 +0,0 @@ -// using System; -// using System.IO; -// using System.Collections.Generic; -// using BCG.Utilities; - -// namespace BCG { -// /// -// /// This is where the all the pubSub State is handled, and its cache kept -// /// Keeps a collection of TagStateBanks to hold the state of each tag -// /// -// public class PubsubServerTagHandler : IPubSubServerTagHandler { -// private IDictionary _tag_state_map = -// new Dictionary(); - -// #region IPubSubServerTagHandler Members - -// public byte[] get_content(string tag) { -// TagStateBank tag_state_bank; -// if (_tag_state_map.TryGetValue(tag, out tag_state_bank)) { -// return tag_state_bank.get_content(); -// } else { -// bts.Log.Gen.warn("Information requested for tag: {tag}, it does not exist now"); -// return new byte[] { }; -// } -// } - -// public void update(string tag, Stream[] messageBodies) { -// bts.Log.Gen.info($"PubsubServerTagHandler.update tag: {tag} streams: {messageBodies.Length}"); -// TagStateBank tag_state_bank; -// if (!_tag_state_map.TryGetValue(tag, out tag_state_bank)) { -// bts.Log.Gen.info("tag: {tag} received for the first time creating TagStateBank for it now"); -// tag_state_bank = new TagStateBank(); -// _tag_state_map.Add(tag, tag_state_bank); -// } -// tag_state_bank.update(messageBodies); -// } - -// public void doWork() { } - -// public string[] supportedTagTypes { get { return new string[] { "DataStreamTag" }; } } - -// public void register(InternalPublishDelegate callback) { } - -// #endregion -// } -// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/StatisticsLogger.cs b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/StatisticsLogger.cs deleted file mode 100644 index f55bd1aa4..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/StatisticsLogger.cs +++ /dev/null @@ -1,102 +0,0 @@ -using System; -using System.IO; -using System.Collections; -using BCG.Utilities; -using System.Threading; - -namespace BCG -{ - class TagData - { - public DateTime lastUpdateTime; - public long totalSize; - public int lastUpdateSize; - - public TagData(DateTime ut, long ts, int lus) - { - lastUpdateTime = ut; - totalSize = ts; - lastUpdateSize = lus; - } - public override string ToString() - { - return lastUpdateTime.ToString() + "," + totalSize.ToString() + "," + lastUpdateSize.ToString(); - } - } - - class StatisticsLogger - { - private IDictionary myTagData = new Hashtable(); - private static StatisticsLogger mySelf; - private BlockingQueue myQueue = new BlockingQueue(); - private DateTime myLastFireTime = DateTime.MinValue; - private TimeSpan myPubsubStatsUpdateInterval = new TimeSpan(0, 0, 0, 0, DataStreamGatewaySettings.instance.pubsubStatsUpdateInterval); - private TimeSpan myTagStatsUpdateInterval = new TimeSpan(0, 0, 0, 0, DataStreamGatewaySettings.instance.singleTagStatsUpdateInterval); - - private StatisticsLogger() - { - Thread readThread = new Thread(new ThreadStart(myReadAndPublish)); - readThread.Priority = ThreadPriority.BelowNormal; - readThread.Start(); - } - public static StatisticsLogger instance - { - get - { - if (mySelf == null) - mySelf = new StatisticsLogger(); - return mySelf; - } - } - - public void log(string tag, Stream[] bodies) - { - long nBytes = 0; - foreach (Stream s in bodies) - nBytes += s.Length; - myQueue.enqueue(new object[] { tag, Convert.ToInt32 (nBytes) }); - } - - private void myReadAndPublish() - { - while (true) - { - object qe = myQueue.dequeue(50); - if (qe != null) - { - object[] args = qe as object[]; - string tag = args[0].ToString(); - int dataSize = (int)args[1]; - if (!myTagData.Contains(tag)) - { - myTagData.Add(tag, new TagData(DateTime.Now, Convert.ToInt64(dataSize), dataSize)); - } - else - { - TagData tdToSend = myTagData[tag] as TagData; - tdToSend.lastUpdateSize += dataSize; - tdToSend.totalSize += dataSize; - } - } - myFireMonitors(); - } - } - - private void myFireMonitors() - { - if (DateTime.Now.Subtract(myLastFireTime) >= myPubsubStatsUpdateInterval) - { - foreach (string tag in myTagData.Keys) - { - TagData td = myTagData[tag] as TagData; - if (td.lastUpdateSize > 0 && DateTime.Now.Subtract(td.lastUpdateTime) >= myTagStatsUpdateInterval) - { - td.lastUpdateTime = DateTime.Now; - td.lastUpdateSize = 0; - } - } - myLastFireTime = DateTime.Now; - } - } - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/TagStateBank.cs b/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/TagStateBank.cs deleted file mode 100644 index 5002b3dd5..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/PubsubReceive/TagStateBank.cs +++ /dev/null @@ -1,32 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; - -namespace BCG -{ - public class TagStateBank - { - private PubsubDeserializationController myDeserializationController = new PubsubDeserializationController(); - //private ClientDeserializationController _deserializer = new ClientDeserializationController(); - - public TagStateBank() { } - - public TagStateBank(string tag) - { - bts.Log.Gen.info("TagStateBank created for tag: {tag}"); - } - - public byte[] get_content() - { - return myDeserializationController.getState(); - } - - public void update(Stream[] streams) - { - bts.Log.Gen.info($"TagStateBank.update streams: {streams.Length}"); - byte[] ops; - Array rowCollection; - //myDeserializationController.deserialize(streams, out ops, out rowCollection); - } - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionClientDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionClientDeserializer.cs deleted file mode 100644 index 279594469..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionClientDeserializer.cs +++ /dev/null @@ -1,146 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.IO; -using BCG.Utilities; - -namespace BCG -{ - public class CompressionClientDeserializer : IClientDeserializer - { - protected IBlueTableSchema mySchema; - protected IBlueTableSchema myFirstSchema; - protected IClientRowDeserializer myRowDeserializer; - - public CompressionClientDeserializer() - { - myFirstSchema = null; - mySchema = null; - } - - #region IClientDeserializer Members - - public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) - { - BinaryReader br = new BinaryReader(UtilityMethods.getAggregatedStream(streams)); - // BlueTrace.WriteDeserializeStart(streams.Length, br.BaseStream.Length); - - // emit # of streams and total length - ArrayList ops = new ArrayList(), rows = new ArrayList(); - IBlueRowCollection[] result; - while (true) - { - byte op; - try - { - int value = br.BaseStream.ReadByte(); - if (value == -1) break; - op = (byte)value; - // BlueTrace.WriteDeserializeOperation(op); - } - catch (EndOfStreamException) - { - break; - } - if (op == StreamOpCode.create) - { - br.ReadString(); //pubsubDeserializer - br.ReadString(); //clientDeserializer - mySchema = new BlueTableSchema(); - mySchema.deserialze(br); - //keep track of the first schema you've ever seen, so you know how to reorder columns - if (myFirstSchema == null) - myFirstSchema = mySchema; - - /// decompress here - int compressedMessageSize = br.ReadInt32(); - byte[] compressedData = br.ReadBytes(compressedMessageSize); - BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(compressedData)); - - result = new IBlueRowCollection[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) }; - myEncorporateRows(result[0], op); - ops.Add(op); - rows.Add(result); - } - else - { - if (mySchema != null) - { - if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove) - { - result = new IBlueRowCollection[] { myGetRows(br, op, 1) }; - myEncorporateRows(result[0], op); - ops.Add(op); - rows.Add(result); - } - else if (op == StreamOpCode.multipleChange) - { - int compressedMessageSize = br.ReadInt32(); - byte[] compressedData = br.ReadBytes(compressedMessageSize); - BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(compressedData)); - - result = new IBlueRowCollection[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) }; - myEncorporateRows(result[0], op); - ops.Add(op); - rows.Add(result); - } - else if (op == StreamOpCode.multipleRemoveAdd) - { - int compressedMessageSize = br.ReadInt32(); - byte[] compressedData = br.ReadBytes(compressedMessageSize); - BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(compressedData)); - - result = - new IBlueRowCollection[] { - myGetRows (decompressedBr, StreamOpCode.remove, decompressedBr.ReadInt32 ()), - myGetRows (decompressedBr, StreamOpCode.add, decompressedBr.ReadInt32 ()) - }; - - myEncorporateRows(result[0], StreamOpCode.remove); - myEncorporateRows(result[1], StreamOpCode.add); - ops.Add(op); - rows.Add(result); - } - else - { - // BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error); - } - } - else - { - // BlueEventLog.logEvent("Received a non create operation even though the schema" + - // "has never been created", BlueEventLevel.Warning); - } - } - } - theOps = ops.ToArray(typeof(byte)) as byte[]; - theRows = rows.ToArray(); - // BlueTrace.WriteDeserializeStop(rows.Count); - } - - #endregion - - private IBlueRowCollection myGetRows(BinaryReader br, byte op, int nRows) - { - BlueTable retVal = new BlueTable(mySchema); - - if (myRowDeserializer == null) - myRowDeserializer = new ClientUniversalRowDeserializer(mySchema); - else if (op == StreamOpCode.create) - myRowDeserializer = new ClientReorderingRowDeserializer(myFirstSchema, mySchema); - - for (int i = 0; i < nRows; i++) - { - IBlueRow row; - myRowDeserializer.deserializeRow(null, br, op, out row); - retVal.addRow(row); - } - return retVal; - } - - private void myEncorporateRows(IBlueRowCollection rows, byte op) { } - - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionPubsubDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionPubsubDeserializer.cs deleted file mode 100644 index 813fca93f..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionPubsubDeserializer.cs +++ /dev/null @@ -1,192 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.IO; -using System.Collections; -using BCG.Utilities; - -namespace BCG -{ - public class CompressionPubsubDeserializer : IPubsubDeserializer - { - protected IDictionary myState = new Hashtable(); - protected IBlueTableSchema mySchema; - protected string myClientDeserializerName; //for the getState method - protected IPubsubRowDeserializer myRowDeserializer; - - public CompressionPubsubDeserializer() { } - - #region IPubSubDeserializer Members - public virtual void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) - { - BinaryReader br = new BinaryReader(UtilityMethods.getAggregatedStream(streams)); - ArrayList ops = new ArrayList(), rows = new ArrayList(); - while (true) - { - IDictionary[] result; - byte op; - try - { - int value = br.BaseStream.ReadByte(); - if (value == -1) break; - op = (byte)value; - } - catch (EndOfStreamException) - { - break; - } - if (op == StreamOpCode.create) - { - myState.Clear(); - br.ReadString(); //the pubsub-deserializer name, should be the same as this's name - myClientDeserializerName = br.ReadString(); //the client-deserializer name - (mySchema = new BlueTableSchema()).deserialze(br); - /// deserialize this part - int compressedMessageSize = br.ReadInt32(); - BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(br.ReadBytes(compressedMessageSize))); - result = new IDictionary[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) }; - myEncorporatRows(result[0], op); - ops.Add(op); - rows.Add(result); - } - else - { - if (mySchema != null) - { - if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove) - { - result = new IDictionary[] { myGetRows(br, op, 1) }; - myEncorporatRows(result[0], op); - ops.Add(op); - rows.Add(result); - } - else if (op == StreamOpCode.multipleChange) - { - int compressedMessageSize = br.ReadInt32(); - BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(br.ReadBytes(compressedMessageSize))); - result = new IDictionary[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) }; - myEncorporatRows(result[0], op); - ops.Add(op); - rows.Add(result); - } - else if (op == StreamOpCode.multipleRemoveAdd) - { - ops.Add(op); - int compressedMessageSize = br.ReadInt32(); - BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(br.ReadBytes(compressedMessageSize))); - result = new IDictionary[] { - myGetRows (decompressedBr, StreamOpCode.remove, decompressedBr.ReadInt32 ()), - myGetRows (decompressedBr, StreamOpCode.add, decompressedBr.ReadInt32 ()) - }; - myEncorporatRows(result[0], StreamOpCode.remove); - myEncorporatRows(result[1], StreamOpCode.add); - rows.Add(result); - } - else - { - // BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error); - } - } - else - { - // BlueEventLog.logEvent("Received a non create operation even though the schema" + - // "has never been created", BlueEventLevel.Warning); - } - } - } - theOps = ops.ToArray(typeof(byte)) as byte[]; - theRows = rows.ToArray(); - } - - /// - /// Sends in a create signal, so that new clients can get a create everytime and start - /// afresh from there - /// - /// - public virtual byte[] getState() - { - if (mySchema == null) - return new byte[] { }; - else - { - BinaryWriter bw = new BinaryWriter(new MemoryStream()); - bw.Write(StreamOpCode.create); - bw.Write(GetType().AssemblyQualifiedName); - bw.Write(myClientDeserializerName); - mySchema.serialize(bw); - /// compress it here - - BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream()); - decompressedBw.Write(myState.Count); - foreach (byte[] row in myState.Values) - decompressedBw.Write(row); - decompressedBw.BaseStream.Position = 0; - byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream); - bw.Write(compressedData.Length); - bw.Write(compressedData); - - return (bw.BaseStream as MemoryStream).ToArray(); - } - } - #endregion - - /// - /// Every row that has been deserialized passes through this method, so that it can update the state. - /// - /// - /// - protected virtual void myEncorporatRows(IDictionary rows, byte op) - { - if (op == StreamOpCode.add || op == StreamOpCode.create) - foreach (DictionaryEntry de in rows) - { - if (myState.Contains(de.Key)) - { - // BlueEventLog2.instance().logEvent(String.Format("Duplicate row creation. t: {0} id:{1}", mySchema.tableName, de.Key), BlueEventLog2Level.Fatal); - StringBuilder b = new StringBuilder(); - foreach (string n in mySchema.columnNames) - b.AppendFormat(" {0}", n); - // BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()), BlueEventLog2Level.Fatal); - } - myState.Add(de.Key, de.Value); - } - else if (op == StreamOpCode.remove) - foreach (DictionaryEntry de in rows) - { - if (!myState.Contains(de.Key)) - { - // BlueEventLog2.instance().logEvent(String.Format("Unknown row deletion. t: {0} id:{1}", mySchema.tableName, de.Key), BlueEventLog2Level.Fatal); - StringBuilder b = new StringBuilder(); - foreach (string n in mySchema.columnNames) - b.AppendFormat(" {0}", n); - // BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()), BlueEventLog2Level.Fatal); - } - myState.Remove(de.Key); - } - else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange) - foreach (DictionaryEntry de in rows) - myState[de.Key] = de.Value; - } - - /// - /// Mass deserialization happens here, also since there is no knowledge of the schema - /// - /// - /// - /// - /// - private IDictionary myGetRows (BinaryReader br, byte op, int noOfRows) - { - myRowDeserializer = myRowDeserializer == null ? new PubsubUniversalRowDeserializer (mySchema) : myRowDeserializer; - Hashtable retVal = new Hashtable (); - for (int i = 0; i < noOfRows; i++) - { - int id; object row; - myRowDeserializer.deserializeRow (myState, br, op, out id, out row); - retVal.Add (id, row); - } - return retVal; - } - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionSerializer.cs deleted file mode 100644 index 090263b2d..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Compression/CompressionSerializer.cs +++ /dev/null @@ -1,90 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.IO; - -namespace BCG -{ - public class CompressionSerializer : IBlueDataSerializer - { - private IBlueRowSerializer myRowSerializer; - private string myPubsubDeserializerName = "BCG.CompressionPubsubDeserializer, DataStreamCore", myClientDeserializerName = "BCG.CompressionClientDeserializer"; - - public CompressionSerializer (IBlueTableSchema schema) - { - myRowSerializer = new UniversalRowSerializer (schema); - } - - public void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData) - { - // BlueTrace.WriteSerializeStart(op, rows.Length); - BinaryWriter bw = new BinaryWriter(new MemoryStream()); - if (op == StreamOpCode.add || op == StreamOpCode.remove || op == StreamOpCode.change) - { - bw.Write(op); - /// no need to compress the only row - myRowSerializer.serializeRow(BlueDataUtility.extractRow(rows[0], 0), op, bw); - } - else if (op == StreamOpCode.create) - { - IBlueRowCollection createdRows = rows[0]; - bw.Write(op); - bw.Write(myPubsubDeserializerName); - bw.Write(myClientDeserializerName); - int dummyIdCol = 0; - int[] dummyIdList = new int[1]; - //the schema doesn't sort anything right now, because the C++ pub sub client depends on things being in a certain order - IBlueTableSchema sortedSchema = BlueDataUtility.getSortedSchema(createdRows.schema, out dummyIdList, out dummyIdCol); - sortedSchema.serialize(bw); - /// compress it here - BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream()); - decompressedBw.Write(createdRows.Count); - foreach (IBlueRow r in createdRows) - myRowSerializer.serializeRow(r, StreamOpCode.add, decompressedBw); - decompressedBw.BaseStream.Position = 0; - - byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream); - bw.Write(compressedData.Length); - bw.Write(compressedData); - } - else if (op == StreamOpCode.multipleRemoveAdd) - { - bw.Write(op); - IBlueRowCollection delRows = rows[0], addRows = rows[1]; - /// compress it here - BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream()); - decompressedBw.Write(delRows.Count); - foreach (IBlueRow dr in delRows) - myRowSerializer.serializeRow(dr, StreamOpCode.remove, decompressedBw); - decompressedBw.Write(addRows.Count); - foreach (IBlueRow ar in addRows) - myRowSerializer.serializeRow(ar, StreamOpCode.add, decompressedBw); - decompressedBw.BaseStream.Position = 0; - - byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream); - bw.Write(compressedData.Length); - bw.Write(compressedData); - } - else if (op == StreamOpCode.multipleChange) - { - bw.Write(op); - IBlueRowCollection changedRows = rows[0]; - /// compress it here - BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream()); - decompressedBw.Write(changedRows.Count); - foreach (IBlueRow cr in changedRows) - myRowSerializer.serializeRow(cr, StreamOpCode.change, decompressedBw); - decompressedBw.BaseStream.Position = 0; - - byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream); - bw.Write(compressedData.Length); - bw.Write(compressedData); - } - bw.BaseStream.Position = 0; - serializedData = (bw.BaseStream as MemoryStream).ToArray(); - // BlueTrace.WriteSerializeStop(serializedData.Length); - } - } - -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/BBOOptionDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/BBOOptionDeserializer.cs deleted file mode 100644 index 2d91628a7..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/BBOOptionDeserializer.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace BCG -{ - public class BBOOptionDeserializer : ClientFixedDeserializer - { - public BBOOptionDeserializer() { } - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/ClientFixedDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/ClientFixedDeserializer.cs deleted file mode 100644 index 6631460da..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/ClientFixedDeserializer.cs +++ /dev/null @@ -1,49 +0,0 @@ -using System; -using BCG.Utilities; - -namespace BCG -{ - public abstract class ClientFixedDeserializer : ClientDeserializer - { - private BlueTable myCurrentState; - - public ClientFixedDeserializer () {} - - protected override IBlueRowCollection myGetRows (System.IO.BinaryReader br, byte op, int nRows) - { - BlueTable retVal = new BlueTable (mySchema); - op = op == StreamOpCode.create ? StreamOpCode.add : - op == StreamOpCode.multipleChange ? StreamOpCode.change : op; - if (myRowDeserializer == null) - { - myRowDeserializer = new ClientFixedRowDeserializer (mySchema); - myCurrentState = new BlueTable (mySchema); - } - for (int i = 0; i < nRows; i++) - { - IBlueRow r; - myRowDeserializer.deserializeRow (myCurrentState, br, op, out r); - retVal.addRow (r); - } - return retVal; - } - - protected override void myEncorporateRows (IBlueRowCollection rows, byte op) - { - if (op == StreamOpCode.create) - myCurrentState = new BlueTable (rows); - else if (op == StreamOpCode.add) - myCurrentState.addRows (rows); - else if (op == StreamOpCode.remove) - myCurrentState.deleteRows (rows); - else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange) - myCurrentState.changeRows (rows); - else - { - // BlueEventLog.logAssert (false, "Illegal operation for this context"); - } - } - - - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/ClientFixedRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/ClientFixedRowDeserializer.cs deleted file mode 100644 index e673a06f7..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/ClientFixedRowDeserializer.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System; -using System.IO; -using BCG.Utilities; - -namespace BCG -{ - public class ClientFixedRowDeserializer : IClientRowDeserializer - { - private PrimitiveDeserializationHelper myHelper; - private IBlueTableSchema mySchema; - public ClientFixedRowDeserializer (IBlueTableSchema schema) - { - myHelper = new PrimitiveDeserializationHelper (mySchema = schema); - } - - #region IClientRowDeserializer Members - - public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) - { - if (op == StreamOpCode.add || op == StreamOpCode.change) - { - object[] retVal = new object[mySchema.columnInfo.Length]; - foreach (ColumnInfo ci in mySchema.columnInfo) - retVal[ci.colNo] = myHelper.deserializeColumn (ci.colNo, br); - r = new BlueRow (mySchema, retVal); - } - else if (op == StreamOpCode.remove) - r = currentState[br.ReadInt32 ()]; - else - { - // BlueEventLog.logAssert (false, op.ToString() + " not valid in this context"); - r = null; - } - } - - #endregion - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/MarketOptionTheoClientDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/MarketOptionTheoClientDeserializer.cs deleted file mode 100644 index 700f7b681..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Client/MarketOptionTheoClientDeserializer.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace BCG -{ - public class MarketOptionTheoClientDeserializer : ClientFixedDeserializer - { - public MarketOptionTheoClientDeserializer() { } - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/BBOOptionSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/BBOOptionSerializer.cs deleted file mode 100644 index c44af6a14..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/BBOOptionSerializer.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace BCG -{ - public class BBOOptionSerializer : FixedSerializer - { - public BBOOptionSerializer(IBlueTableSchema schema) - : base(schema, "BCG.BBOOptionPubsubDeserializer", "BCG.BBOOptionDeserializer") - { } - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/FixedRowSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/FixedRowSerializer.cs deleted file mode 100644 index bc2a09c5f..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/FixedRowSerializer.cs +++ /dev/null @@ -1,35 +0,0 @@ -using System; -using System.IO; - -namespace BCG -{ - /// - /// Optimized to serialize fixed length rows, ie schemas that have only primitive types and non-nulls - /// - public class FixedRowSerializer : IBlueRowSerializer - { - private PrimitiveSerializationHelper myHelper; - private int myIdColNo; - private IBlueTableSchema mySchema; - public FixedRowSerializer (IBlueTableSchema schema) - { - myIdColNo = schema["id"].colNo; - myHelper = new PrimitiveSerializationHelper (schema); - mySchema = schema; - } - - #region IBlueRowSerializer Members - - public void serializeRow (IBlueRow r, byte op, BinaryWriter bw) - { - if (op == StreamOpCode.remove) - myHelper.serializeColumn (r, myIdColNo, bw); - else - foreach (ColumnInfo ci in r.schema.columnInfo) - myHelper.serializeColumn (r, ci.colNo, bw); - } - - - #endregion - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/FixedSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/FixedSerializer.cs deleted file mode 100644 index 064615b0e..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/FixedSerializer.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; -using System.IO; - -namespace BCG -{ - /// - /// Tailored to serialize tables that haved fixed schema (ie no dynamic length types like strings, arrays etc and no nulls) - /// - public abstract class FixedSerializer : BlueDataSerializer - { - public FixedSerializer (IBlueTableSchema schema, - string pubsubDeserializer, - string clientDeserializer) - : base (pubsubDeserializer, clientDeserializer) - { - myRowSerializer = new FixedRowSerializer (schema); - } - } - -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/MarketOptionTheoSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/MarketOptionTheoSerializer.cs deleted file mode 100644 index 719cb20e7..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Publish/MarketOptionTheoSerializer.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace BCG -{ - public class MarketOptionTheoSerializer : FixedSerializer - { - public MarketOptionTheoSerializer(IBlueTableSchema schema) - : base(schema, "BCG.MarketOptionTheoPubsubDeserializer", - "BCG.MarketOptionTheoClientDeserializer") - { } - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/BBOOptionPubsubDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/BBOOptionPubsubDeserializer.cs deleted file mode 100644 index 230d3a8e9..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/BBOOptionPubsubDeserializer.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace BCG -{ - public class BBOOptionPubsubDeserializer : PubsubFixedDeserializer - { - public BBOOptionPubsubDeserializer() { } - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/MarketOptionTheoPubsubDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/MarketOptionTheoPubsubDeserializer.cs deleted file mode 100644 index 0ca4893a5..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/MarketOptionTheoPubsubDeserializer.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace BCG -{ - public class MarketOptionTheoPubsubDeserializer : PubsubFixedDeserializer - { - public MarketOptionTheoPubsubDeserializer() - { } - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/PubsubFixedDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/PubsubFixedDeserializer.cs deleted file mode 100644 index 83e15a2a4..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/PubsubFixedDeserializer.cs +++ /dev/null @@ -1,32 +0,0 @@ -using System; -using System.Collections; -using System.IO; - -namespace BCG -{ - public abstract class PubsubFixedDeserializer : PubsubDeserializer - { - protected BlueTable myCurrentState; - - public PubsubFixedDeserializer () {} - - protected override IDictionary myGetRows (BinaryReader br, byte op, int noOfRows) - { - if (myDeserializer == null) - myDeserializer = new PubsubFixedRowDeserializer (mySchema); - IDictionary retVal = new Hashtable (); - op = op == StreamOpCode.create ? StreamOpCode.add : - op == StreamOpCode.multipleChange ? StreamOpCode.change : op; - for (int i = 0; i < noOfRows; i++) - { - int id; - object row; - myDeserializer.deserializeRow (myState, br, op, out id, out row); - retVal.Add (id, row); - } - return retVal; - } - - } - -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/PubsubFixedRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/PubsubFixedRowDeserializer.cs deleted file mode 100644 index f1e21f641..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLength/Pubsub/PubsubFixedRowDeserializer.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System; -using System.IO; -using System.Collections; - -namespace BCG -{ - public class PubsubFixedRowDeserializer : IPubsubRowDeserializer - { - private int myRowSize, myIdOffset; - - public PubsubFixedRowDeserializer (IBlueTableSchema schema) - { - foreach (ColumnInfo ci in schema.columnInfo) - { - myRowSize += SerializationUtilityMethods.getFixedLengthTypeSize (ci.colType); - myIdOffset += ci.colNo < schema["id"].colNo ? SerializationUtilityMethods.getFixedLengthTypeSize (ci.colType) : 0; - } - } - #region IPubsubRowDeserializer Members - - public void deserializeRow (IDictionary currentState, BinaryReader br, byte op, out int id, out object row) - { - if (op == StreamOpCode.add || op == StreamOpCode.change) - { - row = br.ReadBytes (myRowSize); - id = BitConverter.ToInt32 (row as byte[], myIdOffset); - } - else //delete => only the id - { - id = br.ReadInt32 (); - row = currentState[id]; - } - } - - #endregion - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Client/ClientFixedChangingDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Client/ClientFixedChangingDeserializer.cs deleted file mode 100644 index 069c27e69..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Client/ClientFixedChangingDeserializer.cs +++ /dev/null @@ -1,56 +0,0 @@ -using System; -using System.IO; -using System.Collections; - -using BCG.Utilities; - -namespace BCG -{ - public abstract class ClientFixedChangingDeserializer : ClientDeserializer - { - private string [] myChangingCols; - protected BlueTable myCurrentState; - - public ClientFixedChangingDeserializer (string[] changingCols) - { - myChangingCols = changingCols; - } - - protected override IBlueRowCollection myGetRows (System.IO.BinaryReader br, byte op, int nRows) - { - BlueTable retVal = new BlueTable (mySchema); - op = op == StreamOpCode.create ? StreamOpCode.add : - op == StreamOpCode.multipleChange ? StreamOpCode.change : op; - if (myRowDeserializer == null) - { - myRowDeserializer = new ClientFixedChangingRowDeserializer (mySchema, myChangingCols); - myCurrentState = new BlueTable (mySchema); - } - for (int i = 0; i < nRows; i++) - { - IBlueRow r; - myRowDeserializer.deserializeRow (myCurrentState, br, op, out r); - retVal.addRow (r); - } - return retVal; - } - - protected override void myEncorporateRows (IBlueRowCollection rows, byte op) - { - if (op == StreamOpCode.create) - myCurrentState = new BlueTable (rows); - else if (op == StreamOpCode.add) - myCurrentState.addRows (rows); - else if (op == StreamOpCode.remove) - myCurrentState.deleteRows (rows); - else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange) - myCurrentState.changeRows (rows); - else - { - // BlueEventLog.logAssert (false, "Illegal operation for this context"); - } - } - - - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Client/ClientFixedChangingRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Client/ClientFixedChangingRowDeserializer.cs deleted file mode 100644 index 4e23afd82..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Client/ClientFixedChangingRowDeserializer.cs +++ /dev/null @@ -1,67 +0,0 @@ -using System; -using System.Collections; -using System.IO; - -using BCG.Utilities; - -namespace BCG -{ - /// - /// Deserializes a fixed length row with no nulls on the client side. - /// - public class ClientFixedChangingRowDeserializer : IClientRowDeserializer - { - private int[] myChangingColIndices; - protected PrimitiveDeserializationHelper myHelper; - protected IBlueTableSchema mySchema; - - public ClientFixedChangingRowDeserializer (IBlueTableSchema schema, string[] changingCols) - { - myHelper = new PrimitiveDeserializationHelper (mySchema = schema); - myChangingColIndices = new int [changingCols.Length]; - for (int i = 0; i < changingCols.Length; i++) - myChangingColIndices[i] = schema[changingCols[i]].colNo; - Array.Sort (myChangingColIndices); - } - - public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) - { - if (op == StreamOpCode.add) - myDeserializeAdd (currentState, br, out r); - else if (op == StreamOpCode.remove) - myDeserializeRemove (currentState, br, out r); - else if (op == StreamOpCode.change) - myDeserializeChange (currentState, br, out r); - else - { - r = null; - // BlueEventLog.logAssert (false, "Illegal operation for this function, can only deal with single row " + - // "add, delete and change"); - } - } - - protected virtual void myDeserializeAdd (IBlueRowCollection currentState, BinaryReader br, out IBlueRow r) - { - object[] retVal = new object[mySchema.columnInfo.Length]; - foreach (ColumnInfo ci in mySchema.columnInfo) - retVal[ci.colNo] = myHelper.deserializeColumn (ci.colNo, br); - r = new BlueRow (mySchema, retVal); - } - - protected virtual void myDeserializeRemove (IBlueRowCollection currentState, BinaryReader br, out IBlueRow r) - { - r = currentState[br.ReadInt32()]; - } - - - protected virtual void myDeserializeChange (IBlueRowCollection currentState, BinaryReader br, out IBlueRow r) - { - MutableBlueRow rv = new MutableBlueRow (currentState[br.ReadInt32 ()]); - foreach (int colNo in myChangingColIndices) - rv.setValue (colNo, myHelper.deserializeColumn (colNo, br)); - r = rv; - } - - } - -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Publish/FixedChangingRowSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Publish/FixedChangingRowSerializer.cs deleted file mode 100644 index 3ff9b7268..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Publish/FixedChangingRowSerializer.cs +++ /dev/null @@ -1,90 +0,0 @@ -using System; -using System.Collections; -using System.IO; -using BCG.Utilities; - -namespace BCG -{ - /// - /// Specializes in serializing rows that are sent over as a result of a change; only the changed columns are - /// sent together with the id. - /// - public class FixedChangingRowSerializer : IBlueRowSerializer - { - private int[] myChangingColIndices; - private PrimitiveSerializationHelper myHelper; - private IBlueTableSchema mySchema; - - public FixedChangingRowSerializer (IBlueTableSchema schema, params string[] changingColumns) - { - myHelper = new PrimitiveSerializationHelper (schema); - myChangingColIndices = new int[changingColumns.Length]; - for (int i = 0; i < changingColumns.Length; i++) - myChangingColIndices[i] = schema[changingColumns[i]].colNo; - Array.Sort (myChangingColIndices); - mySchema = schema; - } - - #region IBlueRowSerializer Members - - public void serializeRow (IBlueRow r, byte op, BinaryWriter bw) - { - if (op == StreamOpCode.add) - { - mySerializeAdd (r, bw); - } - else if (op == StreamOpCode.remove) - { - mySerializeRemove (r, bw); - } - else if (op == StreamOpCode.change) - { - mySerializeChange (r, bw); - } - else - { - // BlueEventLog.logAssert (false, String.Format ("{0} is not a valid op in this context", op)); - } - } - - #endregion - - /// - /// The whole row is written, one column after the other with the order defined by the schema. There - /// are no columns headers - /// - /// - /// - protected virtual void mySerializeAdd (IBlueRow r, BinaryWriter bw) - { - foreach (ColumnInfo ci in r.schema.columnInfo) - myHelper.serializeColumn (r, ci.colNo, bw); - } - - /// - /// Only the id is written - /// - /// - /// - protected virtual void mySerializeRemove (IBlueRow r, BinaryWriter bw) - { - myHelper.serializeColumn (r, r.schema["id"].colNo, bw); - } - - /// - /// Serializes only the changed columns, alongside the id. So the format is - /// {id, changedColumns*}. The order of the changedColumns are sorted according - /// to their index in the schema. - /// - /// - /// - protected virtual void mySerializeChange (IBlueRow r, BinaryWriter bw) - { - myHelper.serializeColumn (r, r.schema["id"].colNo, bw); - foreach (int colNo in myChangingColIndices) - myHelper.serializeColumn (r, colNo, bw); - } - - } - -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Publish/FixedChangingSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Publish/FixedChangingSerializer.cs deleted file mode 100644 index 7f61008c4..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Publish/FixedChangingSerializer.cs +++ /dev/null @@ -1,110 +0,0 @@ -using System; -using System.Collections; -using System.IO; -using BCG.Utilities; - -namespace BCG -{ - /// - /// Tailored to serialize tables that haved fixed schema (ie no dynamic length types like strings, arrays etc and no nulls) - /// and are constantly changing - /// - public abstract class FixedChangingSerializer : BlueDataSerializer - { - - public FixedChangingSerializer (IBlueTableSchema schema, - string[] changingCols, - string pubsubDeserializer, - string clientDeserializer) - : base (pubsubDeserializer, clientDeserializer) - { - myRowSerializer = new FixedChangingRowSerializer (schema, changingCols); - } - } - - #region BidAsk - - public class BidAskSerializer : FixedChangingSerializer - { - public BidAskSerializer (IBlueTableSchema schema) - : base (schema, - new string[] {"bidAskZone"}, - "BCG.BidAskPubsubDeserializer", - "BCG.BidAskClientDeserializer" - ) - {} - } - - public class BidAskPubsubDeserializer : PubsubFixedChangingDeserializer - { - public BidAskPubsubDeserializer () : base (new string[] {"bidAskZone"}) - {} - } - - public class BidAskClientDeserializer : ClientFixedChangingDeserializer - { - public BidAskClientDeserializer () : base (new string[] {"bidAskZone"}) - {} - } - - #endregion - - #region HHMarketPrice - - public class HHMarketPriceSerializer : FixedChangingSerializer - { - public HHMarketPriceSerializer(IBlueTableSchema schema) - : base(schema, - new string[] { "MarketPrice" }, - "BCG.HHMarketPricePubsubDeserializer", - "BCG.HHMarketPriceClientDeserializer" - ) - { } - } - - public class HHMarketPricePubsubDeserializer : PubsubFixedChangingDeserializer - { - public HHMarketPricePubsubDeserializer() - : base(new string[] { "MarketPrice" }) - { } - } - - public class HHMarketPriceClientDeserializer : ClientFixedChangingDeserializer - { - public HHMarketPriceClientDeserializer() - : base(new string[] { "MarketPrice" }) - { } - } - - #endregion - - #region ThrottledMarketPrice - - public class ThrottledPriceSerializer : FixedChangingSerializer - { - public ThrottledPriceSerializer(IBlueTableSchema schema) - : base(schema, - new string[] { "PRICE" }, - "BCG.ThrottledPricePubsubDeserializer", - "BCG.ThrottledPriceClientDeserializer" - ) - { } - } - - public class ThrottledPricePubsubDeserializer : PubsubFixedChangingDeserializer - { - public ThrottledPricePubsubDeserializer() - : base(new string[] { "PRICE" }) - { } - } - - public class ThrottledPriceClientDeserializer : ClientFixedChangingDeserializer - { - public ThrottledPriceClientDeserializer() - : base(new string[] { "PRICE" }) - { } - } - - #endregion - -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Pubsub/PubsubFixedChangingDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Pubsub/PubsubFixedChangingDeserializer.cs deleted file mode 100644 index 532cd8912..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Pubsub/PubsubFixedChangingDeserializer.cs +++ /dev/null @@ -1,41 +0,0 @@ -using System; -using System.IO; -using System.Collections; - -using BCG.Utilities; - -namespace BCG -{ - /// - /// Pairs up with FixedSerializer to deserialize messages in the pubsub. - /// - public abstract class PubsubFixedChangingDeserializer : PubsubDeserializer - { - private string[] myChangingCols; - protected BlueTable myCurrentState; - - public PubsubFixedChangingDeserializer (string[] changingColumns) - { - myChangingCols = changingColumns; - } - - protected override IDictionary myGetRows (BinaryReader br, byte op, int noOfRows) - { - if (myDeserializer == null) - myDeserializer = new PubsubFixedChangingRowDeserializer (mySchema, myChangingCols); - IDictionary retVal = new Hashtable (); - op = op == StreamOpCode.create ? StreamOpCode.add : - op == StreamOpCode.multipleChange ? StreamOpCode.change : op; - for (int i = 0; i < noOfRows; i++) - { - int id; - object row; - myDeserializer.deserializeRow (myState, br, op, out id, out row); - retVal.Add (id, row); - } - return retVal; - } - - } - -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Pubsub/PubsubFixedChangingRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Pubsub/PubsubFixedChangingRowDeserializer.cs deleted file mode 100644 index cfce2c24b..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/FixedLengthChangeProne/Pubsub/PubsubFixedChangingRowDeserializer.cs +++ /dev/null @@ -1,84 +0,0 @@ -using System; -using System.IO; -using System.Collections; - -using BCG.Utilities; - -namespace BCG -{ - - /// - /// Pairs up with FixedSerializer to deserialize messages in the pubsub. - /// - public class PubsubFixedChangingRowDeserializer : IPubsubRowDeserializer - { - private int myFullRowSize; - private PrimitiveDeserializationHelper myDeserHelper; - private PrimitiveSerializationHelper mySerHelper; - private IBlueTableSchema mySchema; - private int[] myOffsets, myChangedColIndices; - private int myIdIndex; - - public PubsubFixedChangingRowDeserializer (IBlueTableSchema schema, params string[] changingColNames) - { - myDeserHelper = new PrimitiveDeserializationHelper (mySchema = schema); - mySerHelper = new PrimitiveSerializationHelper (mySchema); - myIdIndex = mySchema["id"].colNo; - myOffsets = new int[schema.columnInfo.Length]; - //size calulcations - - foreach (ColumnInfo ci in mySchema.columnInfo) - { - myOffsets[ci.colNo] += myFullRowSize; - myFullRowSize += SerializationUtilityMethods.getFixedLengthTypeSize (ci.colType); - } - myChangedColIndices = new int[changingColNames.Length]; - for (int i = 0; i < changingColNames.Length; i++) - myChangedColIndices[i] = mySchema[changingColNames[i]].colNo; - Array.Sort (myChangedColIndices); - } - - #region IPubsubRowDeserializer Members - public void deserializeRow (IDictionary currentState, BinaryReader br, byte op, out int id, out object row) - { - if (op == StreamOpCode.add) - myDeserializeAdd (currentState, br, out id, out row); - else if (op == StreamOpCode.remove) - myDeserializeRemove (currentState, br, out id, out row); - else if (op == StreamOpCode.change) - myDeserializeChange (currentState, br, out id, out row); - else - { - id = int.MinValue; - row = null; - // BlueEventLog.logAssert (false, "Illegal operation for this context"); - } - } - #endregion - - protected virtual void myDeserializeAdd (IDictionary currentState, BinaryReader br, out int id, out object row) - { - row = br.ReadBytes (myFullRowSize); - id = BitConverter.ToInt32 (row as byte[], myOffsets[myIdIndex]); - } - - protected virtual void myDeserializeRemove (IDictionary currentState, BinaryReader br, out int id, out object row) - { - id = br.ReadInt32 (); - row = currentState[id]; - } - - protected virtual void myDeserializeChange (IDictionary currentState, BinaryReader br, out int id, out object row) - { - id = br.ReadInt32 (); - BinaryWriter bw = new BinaryWriter (new MemoryStream (currentState[id] as byte[])); - foreach (int colIndex in myChangedColIndices) - { - object newValue = myDeserHelper.deserializeColumn (colIndex, br); - bw.BaseStream.Position = myOffsets[colIndex]; - mySerHelper.serializeColumn (newValue, colIndex, bw); - } - row = (bw.BaseStream as MemoryStream).ToArray (); - } - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientDeserializer.cs deleted file mode 100644 index 555005587..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientDeserializer.cs +++ /dev/null @@ -1,171 +0,0 @@ -// using System; -// using System.Collections; -// using System.IO; -// using System.IO.Compression; -// using BCG.Utilities; -// //using BCG.Utilities.BlueConfiguration; - -// namespace BCG -// { -// public class LookupTableClientDeserializer : IClientDeserializer -// { -// public LookupTableClientDeserializer() { - -// mySchema = new LookupTableSchema(); -// myLookupTables = null; -// } - -// private IClientRowDeserializer myRowDeserializer; -// private LookupTableSchema mySchema; -// private LookupTable[] myLookupTables; - -// public BinaryReader myGetReaderForRestOfStream(Stream theStream) -// { -// //teh rest of the stream is only compressed data -// long compressedBytesLength = theStream.Length - theStream.Position; -// byte[] compressedBytes = new byte[compressedBytesLength]; -// theStream.Read(compressedBytes, 0, (int)compressedBytesLength); - - -// //get a stream to read the uncompressed data -// SharpZipCompressor deKompressor = new SharpZipCompressor(); - -// MemoryStream decompressed = new MemoryStream(); -// deKompressor.decompress(new MemoryStream(compressedBytes), decompressed); -// decompressed.Position = 0; - -// return new BinaryReader(decompressed); -// } -// #region IClientDeserializer methods - -// public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) -// { -// ArrayList ops = new ArrayList(); -// ArrayList rows = new ArrayList(); -// IBlueRowCollection[] result; -// foreach (Stream theStream in streams) -// { -// byte op; - -// BinaryReader br = new BinaryReader(theStream); - -// op = br.ReadByte(); - -// if (op == StreamOpCode.create) -// { -// br.ReadString(); //pubsub deserializer -// br.ReadString(); //client deserializer - - -// br = myGetReaderForRestOfStream(theStream); - -// result = new IBlueRowCollection[] { myHandleCreate(br) }; -// ops.Add(op); -// rows.Add(result); - -// } -// else -// { -// //the schema and lookup tables should already be created. We just need to update everything. -// //figure out how long the rest of the compressed data is -// br = myGetReaderForRestOfStream(theStream); - -// if (mySchema != null) -// { -// if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove) -// { -// myDeserializeLookupTableUpdates(br); -// result = new IBlueRowCollection[] { myGetRows(br, op, 1) }; -// ops.Add(op); -// rows.Add(result); -// } -// else if (op == StreamOpCode.multipleChange) -// { -// myDeserializeLookupTableUpdates(br); -// int numRows = br.ReadInt32(); -// result = new IBlueRowCollection[] { myGetRows(br, op, numRows) }; -// ops.Add(op); -// rows.Add(result); - -// } -// else if (op == StreamOpCode.multipleRemoveAdd) -// { -// myDeserializeLookupTableUpdates(br); -// result = new IBlueRowCollection[2]; -// int numDeletedRows = br.ReadInt32(); -// result[0] = myGetRows(br, StreamOpCode.remove, numDeletedRows); -// int numAddedRows = br.ReadInt32(); -// result[1] = myGetRows(br, StreamOpCode.add, numAddedRows); -// ops.Add(op); -// rows.Add(result); -// } -// else{} -// // BlueEventLog.logEvent("Illegal op-code recieved", BlueEventLevel.Error); - - -// } -// else -// { -// // BlueEventLog.logEvent("Received a non create operation even though the schema" + -// // "has never been created", BlueEventLevel.Error); -// } - - -// } - -// } - -// theOps = ops.ToArray(typeof(byte)) as byte[]; -// theRows = rows.ToArray(); - -// } - - -// #endregion - -// IBlueRowCollection myHandleCreate(BinaryReader br) -// { - -// mySchema.deserialze(br); - -// //build the lookup tables now -// // There is one less than the length of the lookupTableGroups because the first one -// //is always the default group which isn't stored in a lookup table. -// myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1]; - -// //start with 1 because we don't store the 0th lookup table -// for (int i = 0; i < myLookupTables.Length; ++i) -// myLookupTables[i] = new LookupTable(mySchema, i+1); - -// myDeserializeLookupTableUpdates(br); -// //now read the rows -// int numRows = br.ReadInt32(); -// return myGetRows(br, StreamOpCode.create, numRows); - - -// } - - -// private IBlueRowCollection myGetRows(BinaryReader br, byte op, int nRows) -// { -// BlueTable retVal = new BlueTable(mySchema); -// myRowDeserializer = myRowDeserializer == null ? new LookupTableClientRowDeserializer(mySchema,myLookupTables) : myRowDeserializer; -// for (int i = 0; i < nRows; i++) -// { -// IBlueRow row; -// myRowDeserializer.deserializeRow(null, br, op, out row); -// retVal.addRow(row); -// } -// return retVal; -// } - - -// private void myDeserializeLookupTableUpdates(BinaryReader br) -// { -// for (int i = 0; i < myLookupTables.Length; ++i) -// myLookupTables[i].deserializeUpdates(br); -// } - - -// } -// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientlRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientlRowDeserializer.cs deleted file mode 100644 index d9426f951..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientlRowDeserializer.cs +++ /dev/null @@ -1,107 +0,0 @@ -// using System; -// using System.Collections; -// using System.IO; - -// using BCG.Utilities; -// //using BCG.Utilities.BlueConfiguration; - -// namespace BCG -// { -// /// -// /// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process -// /// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations) -// /// -// internal class LookupTableClientRowDeserializer : IClientRowDeserializer -// { -// private LookupTableSchema mySchema; -// private LookupTable[] myTables; -// private int myNullMapLength = 0; -// private PrimitiveDeserializationHelper myDeserializationHelper; -// private BitArray myNullMap; -// private int myIdColumn; -// public LookupTableClientRowDeserializer (LookupTableSchema schema, LookupTable[] theTables) -// { -// mySchema = schema; -// myTables = theTables; - -// myDeserializationHelper = new PrimitiveDeserializationHelper (mySchema); -// myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8))); -// myIdColumn = schema["id"].colNo; - -// } - - - -// #region IClientRowDeserializer Members - -// public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) -// { -// int rowLength = br.ReadInt32 (); //length information for the row not needed -// //rad the null map -// myNullMap = new BitArray(br.ReadBytes (myNullMapLength)); -// object[] values = new object [mySchema.columnNames.Length]; -// //now read the id -// values[myIdColumn] = br.ReadInt32(); -// //now read the lookup groups -// for (int groupNo = 1; groupNo < mySchema.lookupGroups.Length; ++groupNo) -// { -// BlueRowLookupGroup thisGroup = mySchema.lookupGroups[groupNo]; -// LookupTable thisTable = myTables[groupNo - 1]; -// //read the id in, depending upon the size -// int idRead = -1; -// switch (thisGroup.tableSize) -// { -// case 8: -// idRead = br.ReadByte(); -// break; -// case 16: -// idRead = br.ReadInt16(); -// break; -// case 32: -// idRead = br.ReadInt32(); -// break; -// default: -// throw new BlueDataException("Error: invalid size " + thisGroup.tableSize.ToString() + " specific for lookup group."); -// } - -// object[] thisGroupValues = thisTable.getValuesForIndex(idRead); - -// if (thisGroupValues == null) -// throw new BlueDataException("Error: Couldnt' find values for index " + idRead.ToString()); -// if (thisGroupValues.Length != thisGroup.groupColNos.Length) -// throw new BlueDataException("Error: Values extracted for index " + idRead.ToString() + " had length " + thisGroupValues.Length.ToString() + " but length " + thisGroup.groupColNos.Length.ToString() + " was expected."); - - -// //insert those values into our object array - -// for (int groupCol = 0; groupCol < thisGroup.groupColNos.Length; ++groupCol) -// { -// int thisColInOutputRow = thisGroup.groupColNos[groupCol]; -// values[thisColInOutputRow] = thisGroupValues[groupCol]; -// } -// } -// for (int colNo = 0; colNo < values.Length; colNo++) -// { - -// if (myNullMap[colNo]) -// { -// values[colNo] = DBNull.Value; -// } -// //if it's non null, not the id column, and not part of a lookup group, get it here. -// else if (mySchema[colNo].lookupTableGroup == 0 && colNo != myIdColumn) -// { - -// values[colNo] = myDeserializationHelper.deserializeColumn(colNo, br); - -// } -// } - - -// r = new BlueRow(mySchema, values); - -// } - -// #endregion -// } - -// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/LookupTable.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/LookupTable.cs deleted file mode 100644 index 0761ae9cc..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/LookupTable.cs +++ /dev/null @@ -1,194 +0,0 @@ -// using System; -// using System.Collections.Generic; -// using System.Collections; -// using System.IO; -// using System.Linq; -// using System.Text; - - -// namespace BCG -// { - -// /* -// * -// * LookupTable -// * -// * This class represents a lookup table used to store values inside data streams. This allows indexes into the table -// * to be passed over the wire, instead of actual values of the rows. -// * -// * Each table can have multiple lookup table groups. This class represents just one of those groups. This means that all the entries -// * contained in the table will have the same schema. -// * -// * -// * -// * */ -// public class LookupTable -// { -// //each of the entries has the same schema - that is, each of the entries contains the same number of values, with the same -// IBlueTableSchema myEntrySchema; -// UniversalRowSerializer mySerializer; -// ClientUniversalRowDeserializer myDeserializer; -// private const int myIdColumn = 0; - -// //the table is only allowed to reach a maximum size based upon the number of bits you give to it. -// //it throws an error if you try to put too much data into it. -// private long myMaxSize; -// private int mySizeInBits; - -// public int sizeInBits -// { -// get { return mySizeInBits; } -// } - -// KeysBasedTable2 myValuesToEntryMap; - -// //keep track of the last time somebody published you, so you know what values to publish. -// int myIdOfLastPublish; - -// int myIdToUse; - -// /* builds the local schema, serializer, and desrializer -// * initializes to an empty table -// */ -// public LookupTable(LookupTableSchema theSchema, int lookupGroupNumber) -// { -// BlueRowLookupGroup group = theSchema.lookupGroups[lookupGroupNumber]; - -// myEntrySchema = group.entrySchema; - -// //compute how much room you have in the table -// mySizeInBits = group.tableSize; -// myMaxSize = (long)(Math.Pow(2, group.tableSize) - 1); - -// myDeserializer = new ClientUniversalRowDeserializer(myEntrySchema); -// mySerializer = new UniversalRowSerializer(myEntrySchema); - - - -// //create the map -// myValuesToEntryMap = new KeysBasedTable2(myEntrySchema, group.groupColNames); -// myIdOfLastPublish = 0; - - - -// } - -// /*Checks to see if the given values are in the table. IF they are in there, it returns the index to those values. -// If they are not present, it adds them to teh table and returns the new index. */ -// public int getIndexForValues(params object[] values) -// { - -// int index = 0; -// //see if the key is present -// IBlueRow thisRowEntry = myValuesToEntryMap.getRow(values); - -// if (thisRowEntry == null) -// { -// //check to see if we even have room in the table -// if (myIdToUse > myMaxSize) -// throw new BlueDataException("Error: LookupTable has reached maximum size. Make a bigger table."); -// //we need to entrer this new value into the table. Create a blue row using the next ID -// //and the values they supplied to us -// object[] thisRowValues = new object[myEntrySchema.columnInfo.Length]; -// index = myIdToUse; -// thisRowValues[myIdColumn] = index; -// myIdToUse ++; -// //now add in the values they supplied -// for (int colNo = myIdColumn+1; colNo < myEntrySchema.columnInfo.Length; ++colNo) -// { -// thisRowValues[colNo] = values[colNo - 1]; - -// } -// IBlueRow newEntryRow = new BlueRow(myEntrySchema, thisRowValues); - -// myValuesToEntryMap.update(newEntryRow); -// } -// else -// index = (int) thisRowEntry["id"]; - -// return index; - -// } - -// //queries the table to get the values that were stored on a given index. -// //returns null if the values are not found -// public object[] getValuesForIndex(int index) -// { -// object[] results = null; - -// //get the row from the table -// if (myValuesToEntryMap.containsID(index)) -// { -// IBlueRow thisRow = myValuesToEntryMap[index]; -// results = new object[myEntrySchema.columnInfo.Length - 1]; - -// //copy the values from the row into the array -// for (int colNo = myIdColumn+1; colNo < myEntrySchema.columnInfo.Length; ++ colNo) -// { -// results[colNo - 1] = thisRow[colNo]; - -// } - -// } - -// //if the index isn't in the tablem, can just return the null value - - -// return results; - -// } - -// public void deserializeUpdates(BinaryReader br) -// { - -// //get the number of updates that are being published. -// int numUpdates = br.ReadInt32(); - -// for (int i = 0; i < numUpdates; ++i) -// { - -// IBlueRow theNewUpdate = null; -// myDeserializer.deserializeRow(null, br, StreamOpCode.create, out theNewUpdate); -// myValuesToEntryMap.update(theNewUpdate); - -// int thisID = (int) theNewUpdate[myIdColumn]; -// if (thisID != myIdToUse) -// throw new BlueDataException("Error: bad serialized update given to lookup table. Expected id " + myIdToUse.ToString() + ", found " + thisID.ToString()); -// ++myIdToUse; - -// } - - -// } - -// public void serializeUpdates(BinaryWriter bw) -// { -// int numUpdates = myIdToUse - myIdOfLastPublish; - -// //serialize the number of updates -// bw.Write(numUpdates); - -// //serialize each update -// for (int i = myIdOfLastPublish; i < myIdToUse; ++i) -// mySerializer.serializeRow(myValuesToEntryMap[i], StreamOpCode.add, bw); - -// //store the last id you published so that next time we don't serialize the same updates -// myIdOfLastPublish = myIdToUse; -// } - -// /*this function serializes EVERYTHING in the table into the writer. This is -// used by the pubsub deserializer, which needs to be able to supply the new state -// to any clients connection. */ -// public void serializeWholeTable(BinaryWriter bw) -// { -// //serialize the number of updates -// bw.Write(myIdToUse); - -// //serialize each update -// for (int i = 0; i < myIdToUse; ++i) -// mySerializer.serializeRow(myValuesToEntryMap[i], StreamOpCode.add, bw); - - -// } -// } -// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/PubSubDeserialization/LookupTablePubSubDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/PubSubDeserialization/LookupTablePubSubDeserializer.cs deleted file mode 100644 index 0eb191733..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/PubSubDeserialization/LookupTablePubSubDeserializer.cs +++ /dev/null @@ -1,226 +0,0 @@ -// using System; -// using System.Collections; -// using System.IO; -// using BCG.Utilities; -// using System.IO.Compression; - -// namespace BCG -// { - -// public class LookupTablePubSubDeserializer : IPubsubDeserializer -// { -// protected IDictionary myState = new Hashtable(); -// protected LookupTableSchema mySchema; -// protected string myClientDeserializerName; //for the getState method -// protected IPubsubRowDeserializer myDeserializer; -// LookupTable[] myLookupTables; - -// public LookupTablePubSubDeserializer() { - -// mySchema = new LookupTableSchema(); - -// } - - -// public BinaryReader myGetReaderForCompressedPortion(Stream theStream) -// { -// //teh rest of the stream is only compressed data -// long compressedBytesLength = theStream.Length - theStream.Position; -// byte[] compressedBytes = new byte[compressedBytesLength]; -// theStream.Read(compressedBytes, 0, (int)compressedBytesLength); - -// //get a stream to read the uncompressed data -// SharpZipCompressor deKompressor = new SharpZipCompressor(); - -// MemoryStream decompressed = new MemoryStream(); -// deKompressor.decompress(new MemoryStream(compressedBytes), decompressed); -// decompressed.Position = 0; - -// return new BinaryReader(decompressed); -// } - -// #region IPubsubDeserializer Members -// public virtual void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) -// { -// ArrayList ops = new ArrayList(), rows = new ArrayList(); -// foreach (Stream theStream in streams) -// { -// BinaryReader br = new BinaryReader(theStream); -// IDictionary[] result; -// byte op; -// op = br.ReadByte(); - - - -// if (op == StreamOpCode.create) -// { -// myState.Clear(); -// br.ReadString(); //the pubsub-deserializer name, should be the same as this's name -// myClientDeserializerName = br.ReadString(); //the client-deserializer name - -// //at this point, the rest of the bytes in the stream are from a GZIP stream -// //so we need to read into a GZipStream and decompress -// //figure out how long the rest of the compressed data is -// br = myGetReaderForCompressedPortion(theStream); - - -// mySchema.deserialze(br); - -// //now build the lookup tables from the knowledge gleaned at the schema -// myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1]; - -// //start with 1 because we don't store the 0th lookup table -// for (int i = 0; i < myLookupTables.Length; ++i) -// myLookupTables[i] = new LookupTable(mySchema, i + 1); - -// myDeserializeLookupTableUpdates(br); - -// int numRows = br.ReadInt32(); -// result = new IDictionary[] { myGetRows(br, op, numRows) }; -// myEncorporatRows(result[0], op); -// ops.Add(op); -// rows.Add(result); -// } -// else -// { -// //figure out how long the rest of the compressed data is -// br = myGetReaderForCompressedPortion(theStream); -// if (mySchema != null) -// { -// if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove) -// { -// myDeserializeLookupTableUpdates(br); -// result = new IDictionary[] { myGetRows(br, op, 1) }; -// myEncorporatRows(result[0], op); -// ops.Add(op); -// rows.Add(result); -// } -// else if (op == StreamOpCode.multipleChange) -// { -// myDeserializeLookupTableUpdates(br); -// int numRows = br.ReadInt32(); -// result = new IDictionary[] { myGetRows(br, op, numRows) }; -// myEncorporatRows(result[0], op); -// ops.Add(op); -// rows.Add(result); -// } -// else if (op == StreamOpCode.multipleRemoveAdd) -// { -// myDeserializeLookupTableUpdates(br); - -// int numRemoved = br.ReadInt32(); -// result = new IDictionary[2]; - -// result[0] = myGetRows(br, StreamOpCode.remove, numRemoved); -// int numAdded = br.ReadInt32(); -// result[1] = myGetRows (br, StreamOpCode.add, numAdded); - -// myEncorporatRows(result[0], StreamOpCode.remove); -// myEncorporatRows(result[1], StreamOpCode.add); -// ops.Add(op); -// rows.Add(result); -// } -// else -// { -// // BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error); -// } -// } -// else -// { -// // BlueEventLog.logEvent("Received a non create operation even though the schema" + -// // "has never been created", BlueEventLevel.Warning); -// } -// } -// } -// theOps = ops.ToArray(typeof(byte)) as byte[]; -// theRows = rows.ToArray(); -// } - -// /// -// /// Sends in a create signal, so that new clients can get a create everytime and start -// /// afresh from there -// /// -// /// -// public virtual byte[] getState() -// { -// if (mySchema == null) -// return new byte[] { }; -// else -// { -// BinaryWriter bw = new BinaryWriter(new MemoryStream()); -// bw.Write(StreamOpCode.create); -// bw.Write(GetType().FullName); -// bw.Write(myClientDeserializerName); - -// //the rest of this data has to be serialized. -// BinaryWriter compressedBw = new BinaryWriter(new MemoryStream()); -// mySchema.serialize(compressedBw); -// //serialize all the tables -// for (int i = 0; i < myLookupTables.Length; ++i) -// myLookupTables[i].serializeWholeTable(compressedBw); -// compressedBw.Write(myState.Count); -// foreach (byte[] row in myState.Values) -// compressedBw.Write(row); - -// //now compress the bytes -// MemoryStream compressedStream = new MemoryStream(); -// SharpZipCompressor Kompressor = new SharpZipCompressor(); -// compressedBw.BaseStream.Position = 0; - -// Kompressor.compress(compressedBw.BaseStream, compressedStream); - -// //append the two streams together -// byte[] output = new byte[bw.BaseStream.Length + compressedStream.Length]; -// bw.BaseStream.Position = 0; -// compressedStream.Position = 0; - - -// bw.BaseStream.Read(output, 0, (int) bw.BaseStream.Length); -// compressedStream.Read(output, (int) bw.BaseStream.Length, (int) compressedStream.Length); - - - -// return output; -// } -// } -// #endregion - -// private void myDeserializeLookupTableUpdates(BinaryReader br) -// { -// for (int i = 0; i < myLookupTables.Length; ++i) -// myLookupTables[i].deserializeUpdates(br); -// } -// /// -// /// Every row that has been deserialized passes through this method, so that it can update the state. -// /// -// /// -// /// -// protected virtual void myEncorporatRows(IDictionary rows, byte op) -// { -// if (op == StreamOpCode.add || op == StreamOpCode.create) -// foreach (DictionaryEntry de in rows) -// myState.Add(de.Key, de.Value); -// else if (op == StreamOpCode.remove) -// foreach (DictionaryEntry de in rows) -// myState.Remove(de.Key); -// else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange) -// foreach (DictionaryEntry de in rows) -// myState[de.Key] = de.Value; -// } - -// private IDictionary myGetRows(BinaryReader br, byte op, int noOfRows) -// { -// //the row deserializer doesn't need to know anything about the lookup tables -// myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer(mySchema) : myDeserializer; -// Hashtable retVal = new Hashtable(); -// for (int i = 0; i < noOfRows; i++) -// { -// int id; object row; -// myDeserializer.deserializeRow(myState, br, op, out id, out row); -// retVal.Add(id, row); -// } -// return retVal; -// } - -// } -// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableRowSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableRowSerializer.cs deleted file mode 100644 index e866ac3be..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableRowSerializer.cs +++ /dev/null @@ -1,211 +0,0 @@ -// using System; -// using System.Collections; -// using System.IO; -// using BCG.Utilities; -// using System.Collections.Generic; -// namespace BCG -// { -// /// -// /// Helps the LookupTableRowSerializer by serializing one row. -// /// -// /// It does NOT sort the schema, even though the sorted schema doesn't do anything. -// /// That is because the schema sorting utility is agnostic of schema type, and so it would destory the grouping information -// /// which is VITAL -// /// -// /// It performs a mapping : (row)->byte[] -// /// -// /// Row serialized in the following order -// /// row size (nullBitMap size + actual row size) -// /// null bit map -// /// row id -// /// row lookup group ids -// /// rest of row values -// /// -// /// -// /// -// public void serializeRow(IBlueRow r, byte op, BinaryWriter bw) -// { -// myNullMap.SetAll(false); -// int sz = myFixedSize; -// //first, build the null map -// for (int i = 0; i < mySchema.columnNames.Length; i++) -// { -// if (r[i] == DBNull.Value || r[i] == null) -// { -// myNullMap[i] = true; - -// //if this thing isn't in a lookup group, it's no longer going to be accounted for in the fixed size of the table -// //therefore we need to decrement the size of this row by the size of the type -// if (mySchema[i].lookupTableGroup == 0) -// { -// if (SerializationUtilityMethods.isFixedLengthType(mySchema[i].colType)) -// sz -= SerializationUtilityMethods.getFixedLengthTypeSize(mySchema[i].colType); -// } -// } - -// } -// //now serialize the null map -// myNullMap.CopyTo(myNullMapArray, 0); -// int thisRowSize = myGetRowSizeInBytes(sz, r); -// bw.Write(thisRowSize); -// //tehcnically the size calculatinos are all off by 4 becuase tehy don't account for integer telling how big the row is -// long startPos = bw.BaseStream.Position; -// bw.Write(myNullMapArray); -// //write the ID first, for easy access -// bw.Write((int)r[myIdColumn]); -// //now write all the value stored in lookup table groups -// //we don't do this with the first lookup group becuase it's the default one that isn't stored in lookup tables -// for (int groupNo = 1; groupNo < mySchema.lookupGroups.Length; ++groupNo) -// { -// BlueRowLookupGroup thisGroup = mySchema.lookupGroups[groupNo]; -// LookupTable thisTable = myTables[groupNo - 1]; -// int thisLookupID = thisTable.getIndexForValues(thisGroup.extractGroupFromRow(r)); - -// //now, depending on the group size, write this value as either a byte, a short, or an int. - -// switch (thisGroup.tableSize) -// { -// case 8: -// bw.Write((byte)thisLookupID); -// break; -// case 16: -// bw.Write((short)thisLookupID); -// break; -// case 32: -// bw.Write(thisLookupID); -// break; -// default: -// throw new BlueDataException("Error: invalid size " + thisGroup.tableSize.ToString() + " specified for lookup group."); - -// } - -// } -// //now write the rest of the individual columns -// foreach (ColumnInfo ci in mySchema.columnInfo) -// //we have already serialized the id column, as well as all the column groups, and we don't serialize any columns that are null -// if ( (!myNullMap[ci.colNo]) -// && ci.colNo != myIdColumn -// && ci.lookupTableGroup == 0) -// { - - -// myPrimitiveSerializationHelper.serializeColumn(r, ci.colNo, bw); - -// } - -// long stopPos = bw.BaseStream.Position; - -// if (stopPos- startPos != thisRowSize) -// throw new BlueDataException("Error: Bad Size Calculation in LookupTableRowSerializer. Expected size was "+thisRowSize.ToString()+", actual size was "+(stopPos-startPos).ToString()); -// } - -// public void serializeSchema(BinaryWriter bw) -// { -// mySchema.serialize(bw); - -// } -// #endregion - - - - - -// #region size calculations -// /// -// /// Sets up the fixed length (size in bytes occupied by each of the primitive type) -// /// and an index for the non primitive type -// /// -// private void myInitSizeCalculations() -// { -// ArrayList temp = new ArrayList(); -// myFixedSize = myNullMapArray.Length; -// //go through all the columns that aren't in lookup groups, and add up their sizes -// foreach (ColumnInfo ci in mySchema.columnInfo) -// { -// //if you're in a lookup table group, your size is always fixed. -// if (ci.lookupTableGroup == 0) -// { -// if (SerializationUtilityMethods.isFixedLengthType(ci.colType)) -// myFixedSize += SerializationUtilityMethods.getFixedLengthTypeSize(ci.colType); -// else -// temp.Add(ci.colNo); -// } -// } -// //now add on the size of the lookup groups -// foreach (BlueRowLookupGroup group in mySchema.lookupGroups) -// myFixedSize += (group.tableSize / 8); - -// myNonFixedColIndexes = temp.ToArray(typeof(int)) as int[]; -// } - -// /// -// /// Find the size of the serialized row, assumes that nullMap has already -// /// been built -// /// -// /// -// /// -// /// -// private int myGetRowSizeInBytes(int fixedSize, IBlueRow r) -// { -// foreach (int colNo in myNonFixedColIndexes) -// { -// if (!myNullMap[colNo]) -// fixedSize += //myGetStringSizeInBytes (r[colNo] as string); -// SerializationUtilityMethods.getObjectSerializedSize(r[colNo]); -// } -// return fixedSize; -// } -// #endregion -// } -// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableSerializer.cs deleted file mode 100644 index adb17f6e9..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableSerializer.cs +++ /dev/null @@ -1,279 +0,0 @@ -// using System; -// using System.IO; -// using BCG.Utilities; -// using System.Collections.Generic; -// using System.IO.Compression; - - -// namespace BCG -// { -// /// -// /// Serializes data stored in all IBlueRowCollections that use a LookupTableSchema. -// /// -// public class LookupTableSerializer : IBlueDataSerializer -// { -// private const string myPubSubDeser = "BCG.LookupTablePubSubDeserializer", -// myClientDeser = "BCG.LookupTableClientDeserializer"; - - -// private LookupTable[] myLookupTables; -// private LookupTableSchema mySchema; -// private LookupTableRowSerializer myRowSerializer; - -// //the header writer writes into a regular stream -// private BinaryWriter myHeaderWriter; -// //the body writer writes into a compressed binary stream -// //that way, we shrink the size of the data that we send over the wire -// private BinaryWriter myBodyWriter; -// public LookupTableSerializer (IBlueTableSchema inSchema, object[] groups) -// { -// //get the column infos from the input schema -// ColumnInfo[] inputColumns = inSchema.columnInfo; - -// //build the lookup table groups -// //there is one more group than input string because we have the 'default' group, which isn't stored in a table -// int numGroups = groups.Length + 1; -// BlueRowLookupGroup[] lookupGroups = new BlueRowLookupGroup[numGroups]; -// //we start the groups with group no 1, becuase groups with 0 are not stored in lookup tables -// short groupNo = 1; -// Dictionary colNameToGroupNo = new Dictionary(); - -// foreach (string group in groups) -// { -// string[] sizeAndColumns = group.Split(';'); - -// if (sizeAndColumns.Length != 2) -// throw new ArgumentException("Error, bad Lookup Table Group found: " + group); - -// short tableSize = short.Parse(sizeAndColumns[0]); - -// string[] columns = sizeAndColumns[1].Split(','); - -// BlueRowLookupGroup thisGroup = new BlueRowLookupGroup(groupNo, tableSize, columns); -// lookupGroups[groupNo] = thisGroup; -// foreach (string column in columns) -// colNameToGroupNo[column] = groupNo; - -// ++groupNo; - -// } - -// //make the lookup group correpsonding to columns that aren't stored in the lookup table -// List nonLookupTableColumns = new List(); -// foreach (ColumnInfo ci in inputColumns) -// { -// if (!colNameToGroupNo.ContainsKey(ci.colName)) -// { -// nonLookupTableColumns.Add(ci.colName); -// } -// } - -// lookupGroups[0] = new BlueRowLookupGroup(0, 0, nonLookupTableColumns.ToArray()); - - - -// //now build new column infos to replace the old ones -// int numCols = inputColumns.Length; -// ColumnInfo[] newColumns = new ColumnInfo[numCols]; - -// for (int col = 0; col < numCols; ++col) -// { -// ColumnInfo inCol = inputColumns[col]; -// if (inCol.isConstant) -// newColumns[col] = new ColumnInfo(inCol.colName, col, inCol.colType, inCol.colValue); -// else -// { -// string colName = inCol.colName; -// if (!colNameToGroupNo.ContainsKey(inCol.colName)) -// newColumns[col] = new ColumnInfo(colName, col, inCol.colType); -// else -// newColumns[col] = new ColumnInfo(colName, col, inCol.colType, colNameToGroupNo[colName]); -// } - -// } - -// //now build the schema -// mySchema = new LookupTableSchema(inSchema.tableName, newColumns, lookupGroups); - - -// //build the lookup tables. There is one less than the length of the lookupTableGroups because the first one -// //is always the default group which isn't stored in a lookup table. -// myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1]; - -// //start with 1 because we don't store the 0th lookup table -// for (int i = 0; i < myLookupTables.Length; ++i) -// { -// myLookupTables[i] = new LookupTable(mySchema, i+1); -// } - -// //now create the row serializer -// myRowSerializer = new LookupTableRowSerializer(mySchema, myLookupTables); - -// } - -// #region IBlueDataSerializer Members -// /// -// /// Serializes the data related to a single op. -// /// Supports All Ops -// /// -// /// First, a LookupTableUpdate, which is contained in each op, looks like this: -// /// -// /// -// /// -LookupTableUpdate: -// /// group 1 values size \ -// /// group 1 value |- all serialized by a single LookupTableObject -// /// group 1 value / -// /// .... -// /// group 2 values size -// /// group 2 value key -// /// .... -// /// -// /// -// /// Next, the stream formats of the different op codes -// /// -// /// -// /// -create : -// /// op -// /// clientDeser name -// /// pubsubsDeser name -// /// schema (which is a LookupTableSchema) -// /// LookupTableUpdate -// /// num rows -// /// row 1 -// /// row 2 -// /// .... -// /// -// /// -// /// - add, remove, change: -// /// op -// /// LookupTableUpdate -// /// the row -// /// -// /// -multipleRemoveAdd : -// /// op -// /// LookupTableUpdate -// /// num deleted rows -// /// deleted row 1 -// /// deleted row 2 -// /// .... -// /// num added rows -// /// added row 1 -// /// added row 2 -// /// ... -// /// -// /// -multipleChange -// /// op -// /// LookupTableUpdate -// /// num changed rows -// /// changd row 1 -// /// changed row 2 -// /// ... -// /// changd row n -// /// -// /// -// /// -// /// -// /// -// public void serialize(byte op, BCG.IBlueRowCollection[] rows, out byte[] serializedData) -// { -// myHeaderWriter = new BinaryWriter(new MemoryStream()); -// myBodyWriter = new BinaryWriter(new MemoryStream()); -// //everything start with an op -// myHeaderWriter.Write(op); - -// /*go through each of the rows we're about the serialize, and populate the lookup table -// * that way, when we serialize the lookup tables, well have something to report*/ -// foreach (IBlueRowCollection collection in rows) -// foreach (IBlueRow r in collection) -// myRowSerializer.updateLookupTables(r); - -// if (op == StreamOpCode.add || op == StreamOpCode.remove || op == StreamOpCode.change) -// mySerializeSingleRow(op, BlueDataUtility.extractRow(rows[0], 0)); -// else if (op == StreamOpCode.create) -// mySerializeCreate(rows[0]); -// else if (op == StreamOpCode.multipleChange) -// mySerializeMultipleChange(rows[0]); -// else if (op == StreamOpCode.multipleRemoveAdd) -// mySerializeMultipleRemoveAdd(rows[0], rows[1]); -// else -// { -// serializedData = null; -// // BlueEventLog.logAssert(false, String.Format("{0} is not a valid op", op)); -// } - - - -// //copy the data from the header and the body of the message -// myHeaderWriter.BaseStream.Position = 0; -// myBodyWriter.BaseStream.Position = 0; -// int headerLength = (int) myHeaderWriter.BaseStream.Length; - - - -// MemoryStream zippedMemStream = new MemoryStream(); -// SharpZipCompressor Kompressor = new SharpZipCompressor(); - -// Kompressor.compress(myBodyWriter.BaseStream,zippedMemStream); -// byte[] compressedBytes = zippedMemStream.ToArray(); - - -// serializedData = new byte[headerLength + zippedMemStream.Length]; - -// Console.WriteLine(String.Format("Lookuptable Serializer: Header is {0}. Body compresesd from {1} to {2} bytes.", headerLength , myBodyWriter.BaseStream.Length, zippedMemStream.Length)); - - -// myHeaderWriter.BaseStream.Read(serializedData, 0, headerLength); -// BinaryWriter sizeWriter = new BinaryWriter(new MemoryStream(serializedData)); -// zippedMemStream.Position = 0; -// zippedMemStream.Read(serializedData, headerLength, (int) zippedMemStream.Length); - - -// } -// #endregion - -// void mySerializeSingleRow(byte op, IBlueRow row) -// { -// mySerializeLookupTableUpdates(myBodyWriter); -// myRowSerializer.serializeRow(row, op, myBodyWriter); -// } - -// private void mySerializeCreate (IBlueRowCollection rows) -// { -// myHeaderWriter.Write(myPubSubDeser); -// myHeaderWriter.Write(myClientDeser); -// mySchema.serialize(myBodyWriter); -// mySerializeLookupTableUpdates(myBodyWriter); -// myBodyWriter.Write(rows.Count); -// foreach (IBlueRow r in rows) -// myRowSerializer.serializeRow(r, StreamOpCode.create, myBodyWriter); -// } - -// private void mySerializeMultipleChange(IBlueRowCollection rows) -// { -// mySerializeLookupTableUpdates(myBodyWriter); -// myBodyWriter.Write(rows.Count); -// foreach (IBlueRow r in rows) -// myRowSerializer.serializeRow(r,StreamOpCode.change,myBodyWriter); - -// } - -// private void mySerializeMultipleRemoveAdd(IBlueRowCollection deletedRows, IBlueRowCollection addedRows) -// { -// mySerializeLookupTableUpdates(myBodyWriter); -// myBodyWriter.Write(deletedRows.Count); -// foreach (IBlueRow r in deletedRows) -// myRowSerializer.serializeRow(r, StreamOpCode.change, myBodyWriter); -// myBodyWriter.Write(addedRows.Count); -// foreach (IBlueRow r in addedRows) -// myRowSerializer.serializeRow(r, StreamOpCode.add, myBodyWriter); - -// } - -// private void mySerializeLookupTableUpdates(BinaryWriter bw) -// { -// for (int i = 0; i < myLookupTables.Length; ++i) -// myLookupTables[i].serializeUpdates(bw); -// } - -// } -// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/ClientDeserialization/ClientTheoDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/ClientDeserialization/ClientTheoDeserializer.cs deleted file mode 100644 index 2d46651b7..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/ClientDeserialization/ClientTheoDeserializer.cs +++ /dev/null @@ -1,69 +0,0 @@ -using System; -using System.IO; - -namespace BCG -{ - public class ClientTheoDeserializer : IClientDeserializer - { - private IBlueTableSchema mySchema = null; - - public ClientTheoDeserializer() {} - - #region IClientDeserializer Members - - /// - /// Deserialized the streams of bytes, as one operation and the resulting rows. The op - /// is either a create or multipleChange. - /// - /// - /// - /// - public void deserialize (Stream[] streams, out byte[] theOps, out Array theRows) - { - bool isCreate = false; - BinaryReader br = new BinaryReader (UtilityMethods.getAggregatedStream (streams)); - byte op = Byte.MaxValue; - IBlueRowCollection rows = null; - while (true) - { - try - { - op = br.ReadByte (); - } - catch (EndOfStreamException) - { - break; - } - if (op == StreamOpCode.create) - { - isCreate = true; //we crush all ops (that may be part of this call) but if theres a create we want that to be the - //only op - //ie, we dont want creates to be crushed, as this might lead to clients - //not detecting creates (not sure if its really worth it) - br.ReadString (); //pubsubDeserializer - br.ReadString (); //clientDeserializer - (mySchema = new BlueTableSchema ()) .deserialze (br); - } - rows = myGetRows (br, br.ReadInt32 ()); - } - theOps = new byte[] {isCreate ? StreamOpCode.create : op}; - theRows = new IBlueRowCollection[] {rows}; - } - - private IBlueRowCollection myGetRows (BinaryReader br, int n) - { - IBlueRow[] rows = new IBlueRow[n]; - for (int i = 0; i < n; i++) - rows[i] = new BlueRow (mySchema, new object[]{ - i, - br.ReadByte (), - br.ReadByte (), - new DateTime (br.ReadInt64 ()), - br.ReadDouble () - }); - return new SequentialIDBlueTable (mySchema, rows); - } - - #endregion - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/PubsubDeserialization/PubsubTheoDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/PubsubDeserialization/PubsubTheoDeserializer.cs deleted file mode 100644 index 558205f48..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/PubsubDeserialization/PubsubTheoDeserializer.cs +++ /dev/null @@ -1,69 +0,0 @@ -using System; -using System.IO; -using System.Collections; - -namespace BCG -{ - public class PubsubTheoDeserializer : IPubsubDeserializer - { - private string myClientDeserializerName; - private int myNRows; - private byte[] myState; - private int mySizeOfRow ; - private IBlueTableSchema mySchema; - - public PubsubTheoDeserializer() - { - mySizeOfRow = BlueDataValidTypes.getValueTypeSize (BlueDataValidTypes.LONG) + - BlueDataValidTypes.getValueTypeSize (BlueDataValidTypes.BYTE) + - BlueDataValidTypes.getValueTypeSize (BlueDataValidTypes.DOUBLE) + - BlueDataValidTypes.getValueTypeSize (BlueDataValidTypes.BYTE); - } - #region IPubsubDeserializer Members - - public void deserialize (Stream[] streams, out byte[] theOps, out Array theRows) - { - byte op = Byte.MaxValue; - BinaryReader br = new BinaryReader (UtilityMethods.getAggregatedStream (streams)); - while (true) - { - try - { - op = br.ReadByte (); - } - catch (EndOfStreamException) - { - break; - } - if (op == StreamOpCode.create) - { - br.ReadString (); //pubsubDeser name - myClientDeserializerName = br.ReadString (); - (mySchema = new BlueTableSchema ()).deserialze (br); - } - myState = br.ReadBytes ((myNRows = br.ReadInt32()) * mySizeOfRow); - } - theOps = new byte[] {op}; - theRows = new object[] {myState}; - } - - public byte[] getState() - { - if (myState == null) - return new byte[] {}; - else - { - BinaryWriter bw = new BinaryWriter (new MemoryStream ()); - bw.Write (StreamOpCode.create); - bw.Write (GetType().FullName); - bw.Write (myClientDeserializerName); - mySchema.serialize (bw); - bw.Write (myNRows); - bw.Write (myState); - return (bw.BaseStream as MemoryStream).ToArray (); - } - } - - #endregion - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/Serialization/TheoSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/Serialization/TheoSerializer.cs deleted file mode 100644 index 5bc1a7f1d..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/Theos/Serialization/TheoSerializer.cs +++ /dev/null @@ -1,98 +0,0 @@ -using System; -using System.IO; -using BCG.Utilities; - -namespace BCG -{ - /// - /// Serializes the theos, as one table. This is independent of the operation; we transmit - /// the whole state everytime theres a change of any kind. However the operation is also - /// serialized as part of the data; its eaither a create of batchChange - /// - public class TheoSerializer : IBlueDataSerializer - { - private const string myPubSubDeser = "BCG.PubsubTheoDeserializer", - myClientDeser = "BCG.ClientTheoDeserializer"; - - public TheoSerializer (IBlueTableSchema schema) {} - - public TheoSerializer () {} - - #region IBlueDataSerializer Members - /// - /// Serializes the data related to a single op. The only ops written are either a create - /// or multipleChange - /// -create - /// op - /// clientDeser name - /// pubsubsDeser name - /// schema - /// nRows - /// rows - /// -multipleChange - /// op - /// nRows - /// rows - /// This method extracts the rows accprding to the op, massages the op either to be a create - /// or multipleChange and passes it on to mySerialize - /// - /// - /// - /// - public void serialize (byte op, BCG.IBlueRowCollection[] rows, out byte[] serializedData) - { - IBlueRowCollection totalRows = null; - if (op == StreamOpCode.create || op == StreamOpCode.multipleChange) - totalRows = rows[0]; - else if (op == StreamOpCode.multipleRemoveAdd) - totalRows = rows[1]; - else - // BlueEventLog.logAssert (false, String.Format ("{0} is not a valid op", op)); - op = op == StreamOpCode.multipleRemoveAdd ? StreamOpCode.multipleChange : op; - serializedData = mySerialize (op, totalRows); - } - - /// - /// expects a create or multipleChange. - /// - /// - /// - /// - private byte[] mySerialize (byte op, IBlueRowCollection rows) - { - BinaryWriter bw = new BinaryWriter (new MemoryStream ()); - bw.Write (op); - if (op == StreamOpCode.create) - { - bw.Write (myPubSubDeser); - bw.Write (myClientDeser); - myReorderSchema (rows.schema).serialize (bw); - } - bw.Write (rows.Count); - foreach (IBlueRow r in rows) - { - bw.Write ((byte)r["arrayIndex"]); - bw.Write ((byte)r["cell"]); - bw.Write ((long)((DateTime)r["expiration"]).Ticks); - bw.Write ((double)r["value"]); - } - bw.BaseStream.Position = 0; - return (bw.BaseStream as MemoryStream).ToArray (); - } - - #endregion - - public static IBlueTableSchema getSchema() - {return myReorderSchema(new BlueTableSchema("serializedTheos"));} - - private static IBlueTableSchema myReorderSchema (IBlueTableSchema schema) - { - return new BlueTableSchema (schema.tableName, - new ColumnInfo ("id", 0, typeof(int)), - new ColumnInfo ("arrayIndex", 1, typeof(byte)), - new ColumnInfo ("cell", 2, typeof(byte)), - new ColumnInfo ("expiration", 3, typeof(DateTime)), - new ColumnInfo ("value", 4, typeof(double))); - } - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/AccumulatorFactory.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/AccumulatorFactory.cs similarity index 100% rename from src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/AccumulatorFactory.cs rename to src/DataStreamSuite/DataStreamCore/Messaging/Subscription/AccumulatorFactory.cs diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/ITableAccumulator.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/ITableAccumulator.cs similarity index 100% rename from src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/ITableAccumulator.cs rename to src/DataStreamSuite/DataStreamCore/Messaging/Subscription/ITableAccumulator.cs diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator.cs new file mode 100644 index 000000000..962dca796 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator.cs @@ -0,0 +1,285 @@ +// using System; +// using System.Collections; +// using System.IO; +// using BCG.Utilities; +// using BCG.Utilities.General; +// //using BCG.Utilities.BlueConfiguration; +// using System.Collections.Generic; +// using System.Threading; +// using mbts; +// using System.Linq; +// using mbts.spark; + +// namespace BCG { +// public class TableAccumulator : ITableAccumulator { +// private HighPerformanceTimer _timer = new HighPerformanceTimer(); +// private IClientDeserializer myDeserializer = new ClientDeserializationController(); +// private BlueTable myStateTable; +// private ISubscriberCore mySubscriptionCore; +// private TableUpdateDelegate myOnUpdateDelegate; +// private string myTag; +// private bool myUseIncrementsForMultipleOps = DataStreamGatewaySettings.instance.useIncrementsForMultipleOps; + +// public TableAccumulator(string tag, ISubscriberCore subCore) { +// myTag = tag; +// mySubscriptionCore = subCore; +// } + +// #region ITableAccumulator Members + +// public bool is_active() { return myOnUpdateDelegate != null; } + +// public event TableUpdateDelegate on_update { +// add { +// // BlueEventLog.logEvent ("TableSubscriber being Subscribed for tag :" + myTag, BlueEventLevel.Information); +// if (myOnUpdateDelegate == null) { +// //ie first subscription +// // BlueEventLog.logEvent ("Subscription made to PubSub for tag : " + myTag, BlueEventLevel.Information); +// mySubscriptionCore.subscribe(myTag); +// } +// myOnUpdateDelegate = +// Delegate.Combine(myOnUpdateDelegate, value) as TableUpdateDelegate; +// if (myStateTable != null) +// value(myStateTable, myStateTable, new BlueTable(myStateTable.schema)); +// } +// remove { +// // BlueEventLog.logEvent ("TableSubscriber being Unsubscribed for tag :" + myTag, BlueEventLevel.Information); +// if (myOnUpdateDelegate != null) { +// myOnUpdateDelegate = +// Delegate.Remove(myOnUpdateDelegate, value) as TableUpdateDelegate; +// if (myOnUpdateDelegate == null) { +// //all subscribers gone +// // BlueEventLog.logEvent ("Unubscription made to PubSub for tag : " + myTag, BlueEventLevel.Information); +// mySubscriptionCore.unsubscribe(myTag); +// myStateTable = null; +// } +// } + +// } +// } + +// public void handle_updates(IEnumerable msgs) { +// update(msgs.Select(t => new MemoryStream(t)).ToArray()); +// } + +// public void update(Stream[] streams) { +// bts.Log.Gen.info($"TableAccumulator.update tag: {myTag} n_streams: {streams.Length} stream[0].len: {streams[0].Length - streams[0].Position}"); +// //checking the condition as a subscriber might unsubscribe in a callback and there might be more than one +// //data element for that tag in the ViewStreamDatabase +// if (myOnUpdateDelegate != null) { +// byte[] ops; +// Array rows; +// _timer.start(); +// myDeserializer.deserialize(streams, out ops, out rows); +// _timer.stop(); +// // BlueEventLog.logEvent(String.Format("deserializing {0} took {1} msecs", myTag, _timer.lastIntervalTime), BlueEventLevel.Performance); +// _timer.start(); +// myAccumulateAndFire(ops, rows); +// _timer.stop(); +// // BlueEventLog.logEvent(String.Format("myAccumulateAndFire for {0} took {1} msecs", myTag, _timer.lastIntervalTime), BlueEventLevel.Performance); +// } +// } + +// #endregion + +// private void myAccumulateAndFire(byte[] ops, Array rows) { +// //if a non create comes before the create, the deserializer returns 0 sized arrays +// if (ops.Length == 0 || rows.Length == 0) { +// //ie if one is zero sized so should be the other one +// // BlueEventLog.logAssert (ops.Length + rows.Length == 0); +// } else { +// if (ops.Length > 1) { +// int index = 0; +// if (myStateTable == null) { +// for (index = 0; index < ops.Length; index++) +// if (ops[index] == StreamOpCode.create) +// break; +// if (index == ops.Length) +// return; +// } +// if (myUseIncrementsForMultipleOps) +// myHandleMultipleOpsIncrementally(ops, rows, index); +// else +// myHandleMultipleOps(ops, rows, index); +// } else { +// if (myStateTable != null || ops[0] == StreamOpCode.create) +// myHandleSingleOp(ops, rows); +// } +// } +// } + +// private void myHandleMultipleOps(byte[] ops, Array rows, int offset) { +// HighPerformanceTimer timer = new HighPerformanceTimer(); +// timer.start(); +// IBlueRowCollection stateAtStart = myStateTable == null ? null : new BlueTable(myStateTable); +// for (int i = offset; i < ops.Length; i++) { +// byte op = ops[i]; +// IBlueRowCollection[] tableArray = rows.GetValue(i) as IBlueRowCollection[]; +// if (op == StreamOpCode.add) +// myStateTable.addRows(tableArray[0]); +// else if (op == StreamOpCode.remove) +// myStateTable.deleteRows(tableArray[0]); +// else if (op == StreamOpCode.change) +// myStateTable.changeRows(tableArray[0]); +// else if (op == StreamOpCode.create) +// myStateTable = new BlueTable(tableArray[0]); +// else if (op == StreamOpCode.multipleChange) +// myStateTable.changeRows(tableArray[0]); +// else if (op == StreamOpCode.multipleRemoveAdd) { +// myStateTable.deleteRows(tableArray[0]); +// myStateTable.addRows(tableArray[1]); +// } +// } +// IBlueRowCollection dRows, aRows; +// if (stateAtStart == null) { +// dRows = new BlueTable(myStateTable.schema); +// aRows = myStateTable; +// } else { +// BlueDataUtility.computeDifferences(myStateTable, stateAtStart, out dRows, out aRows); +// } +// timer.stop(); +// //Console.WriteLine("multiple ops for table " + myTag + " took " + timer.lastIntervalTime + " msecs"); +// myOnUpdateDelegate(myStateTable, aRows, dRows); + +// } + +// #region handling mutliple ops in single update call, incrementally + +// private void myHandleMultipleOpsIncrementally(byte[] ops, Array rows, int offset) { +// BlueTable dels = null, adds = null; +// if (myStateTable != null) { +// dels = new BlueTable(myStateTable.schema); +// adds = new BlueTable(myStateTable.schema); +// } +// for (int i = offset; i < ops.Length; i++) { +// byte op = ops[i]; +// IBlueRowCollection[] tableArray = rows.GetValue(i) as IBlueRowCollection[]; +// if (op == StreamOpCode.add) { +// myHandleAdd(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds); +// } else if (op == StreamOpCode.remove) { +// myHandleRemove(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds); +// } else if (op == StreamOpCode.change) { +// myHandleChange(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds); +// } else if (op == StreamOpCode.create) { +// myHandleCreate(tableArray[0], ref dels, ref adds); +// } else if (op == StreamOpCode.multipleChange) { +// myHandleMultipleChanges(tableArray[0], dels, adds); +// } else if (op == StreamOpCode.multipleRemoveAdd) { +// myHandleMultipleRemovesAdds(tableArray[0], tableArray[1], dels, adds); +// } +// } +// myOnUpdateDelegate(myStateTable, adds, dels); +// } + +// private void myHandleAdd(IBlueRow r, BlueTable dels, BlueTable adds) { +// if (dels != null) { +// myStateTable.addRow(r); +// adds.addRow(r); +// } +// } + +// private void myHandleRemove(IBlueRow r, BlueTable dels, BlueTable adds) { +// if (dels != null) { +// myStateTable.deleteRow(r); +// if (adds.containsID((int)r["id"])) +// adds.deleteRow(r); +// else +// dels.addRow(r); +// } +// } + +// private void myHandleChange(IBlueRow r, BlueTable dels, BlueTable adds) { +// if (dels != null) { +// IBlueRow er = myStateTable[(int)r["id"]]; +// myHandleRemove(er, dels, adds); +// myHandleAdd(r, dels, adds); +// } +// } + +// private void myHandleCreate(IBlueRowCollection newTable, ref BlueTable dels, ref BlueTable adds) { +// if (dels == null) { +// // BlueEventLog.logAssert(adds == null); +// dels = new BlueTable(newTable.schema); +// adds = new BlueTable(newTable); +// myStateTable = new BlueTable(newTable); +// } else { +// dels = new BlueTable(myStateTable); +// myStateTable = new BlueTable(newTable); +// adds = new BlueTable(newTable); +// } +// } + +// private void myHandleMultipleRemovesAdds(IBlueRowCollection deletedRows, IBlueRowCollection addedRows, +// BlueTable dels, BlueTable adds) { +// if (dels != null) { +// foreach (IBlueRow r in deletedRows) +// myHandleRemove(r, dels, adds); +// foreach (IBlueRow r in addedRows) +// myHandleAdd(r, dels, adds); +// } +// } + +// private void myHandleMultipleChanges(IBlueRowCollection changedRows, BlueTable dels, BlueTable adds) { +// if (dels != null) +// foreach (IBlueRow r in changedRows) +// myHandleChange(r, dels, adds); +// } + +// #endregion + +// private void myHandleSingleOp(byte[] ops, Array rows) { +// byte op = ops[0]; +// IBlueRowCollection[] tableArray = rows.GetValue(0) as IBlueRowCollection[]; +// IBlueRowCollection dRows, aRows; +// if (op == StreamOpCode.create) { +// if (myStateTable == null) { +// aRows = myStateTable = new BlueTable(tableArray[0]); +// dRows = new BlueTable(myStateTable.schema); +// myOnUpdateDelegate(myStateTable, aRows, dRows); +// } else { +// IBlueRowCollection newState = new BlueTable(tableArray[0]); +// BlueDataUtility.computeDifferences(newState, myStateTable, out dRows, out aRows); +// if (aRows.Count > 0 || dRows.Count > 0) { +// myStateTable = new BlueTable(newState); +// myOnUpdateDelegate(myStateTable, aRows, dRows); +// } +// } +// } else { +// if (op == StreamOpCode.add) { +// myStateTable.addRows(aRows = tableArray[0]); +// dRows = new BlueTable(myStateTable.schema); +// } else if (op == StreamOpCode.remove) { +// myStateTable.deleteRows(dRows = tableArray[0]); +// aRows = new BlueTable(myStateTable.schema); +// } else if (op == StreamOpCode.change) { +// IBlueRow cRow = BlueDataUtility.extractRow(tableArray[0], 0); +// aRows = new BlueTable(myStateTable.schema); +// dRows = new BlueTable(myStateTable.schema); +// (aRows as BlueTable).addRow(cRow); +// (dRows as BlueTable).addRow(myStateTable[(int)cRow["id"]]); +// myStateTable.changeRow(cRow); +// } else if (op == StreamOpCode.multipleChange) { +// if (myStateTable.Count == tableArray[0].Count) { +// //all rows are changed, so we can optimize this +// dRows = myStateTable; +// aRows = myStateTable = new BlueTable(tableArray[0]); +// } else { +// dRows = new BlueTable(myStateTable.schema); +// aRows = tableArray[0]; +// foreach (IBlueRow r in tableArray[0]) { +// (dRows as BlueTable).addRow(myStateTable[(int)r["id"]]); +// myStateTable.changeRow(r); +// } +// } +// } else if (op == StreamOpCode.multipleRemoveAdd) { +// BlueDataUtility.absorbChanges(myStateTable, dRows = tableArray[0], aRows = tableArray[1]); +// } else { +// aRows = dRows = null; +// // BlueEventLog.logAssert(false, "unknown opcode"); +// } +// myOnUpdateDelegate(myStateTable, aRows, dRows); +// } +// } +// } + +// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator2.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs similarity index 100% rename from src/DataStreamSuite/DataStreamCore/Messaging/Accumulation/TableAccumulator2.cs rename to src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Test/CompressionSerializationTest.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Test/CompressionSerializationTest.cs deleted file mode 100644 index 0196ec0ad..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Test/CompressionSerializationTest.cs +++ /dev/null @@ -1,236 +0,0 @@ -// using System; -// using System.IO; -// using NUnit.Framework; - -// namespace BCG -// { - -// /// -// /// Tests the deserialization happening within the pubsub. -// /// -// [TestGroup] -// public class CompressionSerializationTest -// { -// protected IBlueTableSchema mySchema; -// protected BlueTable myTable; -// protected IBlueDataSerializer mySer; -// protected IPubsubDeserializer myPubsubDeser; -// protected IClientDeserializer myClientDeser; - - -// [Test] -// public static void test_compression() -// { -// int n_len = 10000000; -// for (int t = 0; t < 10; t++) -// { -// Random rnd = new Random(DateTime.Now.Millisecond); -// byte[] data = new byte[n_len]; -// rnd.NextBytes(data); - -// BCG.Utilities.General.HighPerformanceTimer timer = new Utilities.General.HighPerformanceTimer(); -// timer.start(); -// byte[] compr = UtilityMethods.compress(new MemoryStream(data)); -// timer.stop(); -// Console.WriteLine("Time to compress {0} bytes {1} msecs ratio {2}", n_len, timer.lastIntervalTime, (1.0 * data.Length)/(1.0 * compr.Length)); - -// timer.start(); -// Stream decompr_str = UtilityMethods.decompress(compr); -// timer.stop(); -// Console.WriteLine("Time to de-compress {0} bytes {1} msecs", n_len, timer.lastIntervalTime); - -// decompr_str.Position = 0; -// Assert.AreEqual(data.Length, (int)decompr_str.Length); -// for (int i = 0; i < data.Length; i++) -// Assert.AreEqual(data[i], decompr_str.ReadByte()); -// } -// } - -// public CompressionSerializationTest() -// { -// mySchema = new BlueTableSchema(myGetTableName(), -// new ColumnInfo("id", 0, typeof(int)), -// new ColumnInfo("i1", 1, typeof(int)), -// new ColumnInfo("d1", 2, typeof(double)), -// new ColumnInfo("dt", 3, typeof(DateTime)), -// new ColumnInfo("x", 4, typeof(byte)) -// ); -// mySer = new CompressionSerializer(mySchema); -// myClientDeser = new ClientDeserializationController(); -// myPubsubDeser = new PubsubDeserializationController(); -// } - -// protected virtual string myGetTableName() -// { -// return "testUniversal"; -// } - -// /// -// /// Tests the entire cycle. This is done by 'publishing' data to the serializationController (ie a series of -// /// adds, deletions, changes etc) and getting the bytes and feeding it into a pubsubDeseriazlier and a clientDeserializer. -// /// At every step the data is tested for consistency. At the end the getState is used to get all the data, and its passed -// /// through the clientDeserializer to get all the rows. -// /// -// [Test] -// public void testEntireProcess() -// { -// myTable = new BlueTable(mySchema); -// byte[] data; -// int id; -// //create -// for (id = 0; id < 100; id++) -// myTable.addRow(myGetRow(id)); - -// byte[] ops; -// Array rows; -// IBlueRowCollection del, add; -// IBlueRowCollection[] receivedData; -// IBlueRow r; - -// //create -// mySer.serialize(StreamOpCode.create, new IBlueRowCollection[] { myTable }, out data); -// myWrite(StreamOpCode.create, data); -// myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows); -// myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows); -// Assert.AreEqual(1, ops.Length); -// Assert.AreEqual(StreamOpCode.create, ops[0]); -// Assert.AreEqual(1, rows.Length); -// receivedData = rows.GetValue(0) as IBlueRowCollection[]; -// Assert.AreEqual(1, receivedData.Length); - -// BlueDataUtility.computeDifferences(receivedData[0], myTable, out del, out add); -// Assert.AreEqual(0, del.Count); -// Assert.AreEqual(0, add.Count); - -// //single add -// r = myGetRow(id); -// BlueTable temp = new BlueTable(mySchema); -// myTable.addRow(r); -// temp.addRow(r); -// mySer.serialize(StreamOpCode.add, new IBlueRowCollection[] { temp }, out data); -// myWrite(StreamOpCode.add, data); -// myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows); -// myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows); -// Assert.AreEqual(1, ops.Length); -// Assert.AreEqual(StreamOpCode.add, ops[0]); -// Assert.AreEqual(1, rows.Length); -// receivedData = rows.GetValue(0) as IBlueRowCollection[]; -// Assert.AreEqual(1, receivedData.Length); -// BlueDataUtility.computeDifferences(receivedData[0], temp, out del, out add); -// Assert.AreEqual(0, del.Count); -// Assert.AreEqual(0, add.Count); - -// //single delete -// r = myTable[0]; //delete row with id 0 -// temp = new BlueTable(mySchema); -// myTable.deleteRow(0); -// temp.addRow(r); -// mySer.serialize(StreamOpCode.remove, new IBlueRowCollection[] { temp }, out data); -// myWrite(StreamOpCode.remove, data); -// myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows); -// myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows); -// Assert.AreEqual(1, ops.Length); -// Assert.AreEqual(StreamOpCode.remove, ops[0]); -// Assert.AreEqual(1, rows.Length); -// receivedData = rows.GetValue(0) as IBlueRowCollection[]; -// Assert.AreEqual(1, receivedData.Length); -// BlueDataUtility.computeDifferences(receivedData[0], temp, out del, out add); -// Assert.AreEqual(0, del.Count); -// Assert.AreEqual(0, add.Count); - -// //batch deletes and adds -// //0 was removed earlier, so lets delete from 1 to 24 first, and then add 101 to 124 -// BlueTable delRows = new BlueTable(mySchema), addRows = new BlueTable(mySchema); -// for (int i = 1; i < 25; i++) -// { -// delRows.addRow(myTable[i]); -// myTable.deleteRow(i); -// } -// for (id = 101; id < 125; id++) -// { -// r = myGetRow(id); -// addRows.addRow(r); -// myTable.addRow(r); -// } -// for (int i = 25; i < 125; i++) -// Assert.AreEqual(true, myTable.containsID(i)); -// Assert.AreEqual(100, myTable.Count); - -// mySer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { delRows, addRows }, out data); -// myWrite(StreamOpCode.multipleRemoveAdd, data); -// myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows); -// myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows); -// Assert.AreEqual(1, ops.Length); -// Assert.AreEqual(StreamOpCode.multipleRemoveAdd, ops[0]); -// Assert.AreEqual(1, rows.Length); -// receivedData = rows.GetValue(0) as IBlueRowCollection[]; -// Assert.AreEqual(2, receivedData.Length); -// BlueDataUtility.computeDifferences(receivedData[0], delRows, out del, out add); -// Assert.AreEqual(0, del.Count); -// Assert.AreEqual(0, add.Count); -// BlueDataUtility.computeDifferences(receivedData[1], addRows, out del, out add); -// Assert.AreEqual(0, del.Count); -// Assert.AreEqual(0, add.Count); - -// //pure changes only -// //change the first 25, 25-49 -// BlueTable changedRows = new BlueTable(mySchema); -// for (int i = 25; i < 50; i++) -// { -// MutableBlueRow mr = new MutableBlueRow(myTable[i]); -// mr.setValue(4, Byte.MaxValue); -// myTable.changeRow(mr); -// changedRows.addRow(mr); -// } -// mySer.serialize(StreamOpCode.multipleChange, new IBlueRowCollection[] { changedRows }, out data); -// myWrite(StreamOpCode.multipleChange, data); -// myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows); -// myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows); -// Assert.AreEqual(1, ops.Length); -// Assert.AreEqual(StreamOpCode.multipleChange, ops[0]); -// Assert.AreEqual(1, rows.Length); -// receivedData = rows.GetValue(0) as IBlueRowCollection[]; -// Assert.AreEqual(1, receivedData.Length); -// BlueDataUtility.computeDifferences(receivedData[0], changedRows, out del, out add); -// Assert.AreEqual(0, del.Count); -// Assert.AreEqual(0, add.Count); - -// //final test -// data = myPubsubDeser.getState(); -// new ClientDeserializationController().deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows); -// Assert.AreEqual(1, ops.Length); -// Assert.AreEqual(StreamOpCode.create, ops[0]); -// Assert.AreEqual(1, rows.Length); -// receivedData = rows.GetValue(0) as IBlueRowCollection[]; -// Assert.AreEqual(1, receivedData.Length); -// BlueDataUtility.computeDifferences(receivedData[0], myTable, out del, out add); -// Assert.AreEqual(0, del.Count); -// Assert.AreEqual(0, add.Count); - -// } - -// private IBlueRow myGetRow(int id) -// { -// return new BlueRow(mySchema, // the day pp fixed this test. -// id, id + 10, id * 3.0, new DateTime(2008, 6, 4, 0, 0, 0, DateTimeKind.Unspecified), Convert.ToByte(id)); -// } - -// private void myWrite(byte op, byte[] data) -// { -// string s = ""; -// if (op == StreamOpCode.add) -// s = "add one row"; -// else if (op == StreamOpCode.change) -// s = "change one row"; -// else if (op == StreamOpCode.create) -// s = "create"; -// else if (op == StreamOpCode.multipleChange) -// s = "multiple change"; -// else if (op == StreamOpCode.multipleRemoveAdd) -// s = "multiple add remove"; -// else if (op == StreamOpCode.remove) -// s = "remove one row"; -// Console.WriteLine("op = {0}, size = {1}", s, data.Length); -// } -// } -// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Test/FixedChangingRowSerializationTest.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Test/FixedChangingRowSerializationTest.cs deleted file mode 100644 index 7426db6b5..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Test/FixedChangingRowSerializationTest.cs +++ /dev/null @@ -1,100 +0,0 @@ -// using System; -// using System.IO; -// using System.Collections; -// using NUnit.Framework; - -// namespace BCG -// { -// [TestGroup] -// public class FixedChangingRowSerializationTest -// { -// private IBlueTableSchema mySchema = new BlueTableSchema ("Test", -// new ColumnInfo ("i", 0, typeof(int)), -// new ColumnInfo ("b", 1, typeof(bool)), -// new ColumnInfo ("d", 2, typeof(double)), -// new ColumnInfo ("id", 3, typeof(int)), -// new ColumnInfo ("dt", 4, typeof(DateTime)), -// new ColumnInfo ("bt", 5, typeof(byte))); -// private BlueTable myState; -// private Random myRnd = new Random (DateTime.Now.Millisecond); -// private IBlueRowSerializer mySerializer; -// private IClientRowDeserializer myRowDeserializer; - -// public FixedChangingRowSerializationTest () -// { -// myState = new BlueTable (mySchema); -// mySerializer = new FixedChangingRowSerializer (mySchema, "bt"); -// myRowDeserializer = new ClientFixedChangingRowDeserializer (mySchema, new string[] {"bt"}); -// } - -// [Test] -// public void addTest () -// { -// for (int i = 0; i < 1000; i++) -// { -// BinaryWriter bw = new BinaryWriter (new MemoryStream ()); -// IBlueRow r = myCreateRow (i); -// mySerializer.serializeRow (r, StreamOpCode.add, bw); - -// BinaryReader br = new BinaryReader (new MemoryStream ((bw.BaseStream as MemoryStream).ToArray ())); -// Assert.AreEqual (r["i"], br.ReadInt32()); -// Assert.AreEqual (r["b"], br.ReadBoolean ()); -// Assert.AreEqual (r["d"], br.ReadDouble ()); -// Assert.AreEqual (r["id"], br.ReadInt32 ()); -// Assert.AreEqual (((DateTime)r["dt"]).Ticks, br.ReadInt64()); -// Assert.AreEqual (r["bt"], br.ReadByte ()); -// } -// } - -// [Test] -// public void removeTest () -// { -// for (int i = 0; i < 1000; i++) -// { -// BinaryWriter bw = new BinaryWriter (new MemoryStream ()); -// IBlueRow r = myCreateRow (i); -// mySerializer.serializeRow (r, StreamOpCode.remove, bw); - -// BinaryReader br = new BinaryReader (new MemoryStream ((bw.BaseStream as MemoryStream).ToArray ())); -// Assert.AreEqual (r["id"], br.ReadInt32()); -// } -// } - -// [Test] -// public void changeTest () -// { -// for (int i = 0; i < 1000; i++) -// { -// BinaryWriter bw = new BinaryWriter (new MemoryStream ()); -// IBlueRow r = myCreateRow (i); -// mySerializer.serializeRow (r, StreamOpCode.change, bw); - -// BinaryReader br = new BinaryReader (new MemoryStream ((bw.BaseStream as MemoryStream).ToArray ())); -// Assert.AreEqual (r["id"], br.ReadInt32()); -// Assert.AreEqual (r["bt"], br.ReadByte()); -// bool isEnd = false; -// try -// { -// br.ReadByte (); -// } -// catch (EndOfStreamException) -// { -// isEnd = true; -// } -// Assert.That (isEnd); -// } -// } - -// private IBlueRow myCreateRow (int id) -// { -// return new BlueRow (mySchema, -// myRnd.Next (1000), -// myRnd.Next (1000) % 2 == 0, -// myRnd.NextDouble (), -// id, -// DateTime.Now, -// Convert.ToByte (10)); -// } - -// } -// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Test/FixedChangingSerializationTest.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Test/FixedChangingSerializationTest.cs deleted file mode 100644 index d11c70996..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Test/FixedChangingSerializationTest.cs +++ /dev/null @@ -1,40 +0,0 @@ -// using System; - -// namespace BCG -// { - -// public class TestFixedChangingSerializer : FixedChangingSerializer -// { -// public TestFixedChangingSerializer (IBlueTableSchema schema) -// : base (schema, -// new string[] {"x"}, -// "BCG.PubsubTestFixedChangingDeserializer", -// "BCG.ClientTestFixedChangingDeserializer" -// ) -// {} -// } - -// public class PubsubTestFixedChangingDeserializer : PubsubFixedChangingDeserializer -// { -// public PubsubTestFixedChangingDeserializer () : base (new string[] {"x"}) -// {} -// } - -// public class ClientTestFixedChangingDeserializer : ClientFixedChangingDeserializer -// { -// public ClientTestFixedChangingDeserializer () : base (new string[] {"x"}) -// {} -// } - - -// public class FixedChangingSerializationTest : UniversalSerializationTest -// { -// public FixedChangingSerializationTest () {} - -// protected override string myGetTableName() -// { -// return "testFixedChanging"; -// } - -// } -// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Test/MarketOptionTheoSerializationTest.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Test/MarketOptionTheoSerializationTest.cs deleted file mode 100644 index 5b5a0dae2..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Test/MarketOptionTheoSerializationTest.cs +++ /dev/null @@ -1,73 +0,0 @@ -// using System; -// using System.Collections; -// using System.IO; -// using NUnit.Framework; -// using BCG.Utilities.General; - -// namespace BCG -// { -// [TestFixture] -// public class MarketOptionTheoSerializationTest -// { -// private IBlueTableSchema mySchema = new BlueTableSchema ("marketOptionTheo", -// new ColumnInfo ("arrayIndex", 0, typeof(byte)), -// new ColumnInfo ("cell", 1, typeof(byte)), -// new ColumnInfo ("expiration", 2, typeof(DateTime)), -// new ColumnInfo ("id", 3, typeof(int)), -// new ColumnInfo ("value", 4, typeof(double)) -// ); - -// public MarketOptionTheoSerializationTest () {} - -// [Test] -// public void serializationPerformanceTest () -// { -// MarketOptionTheoSerializer ser = new MarketOptionTheoSerializer (mySchema); -// IClientDeserializer deser = new MarketOptionTheoClientDeserializer (); -// BlueTable bt = new BlueTable (mySchema); -// int count = 3000; -// byte[] ops; -// Array rows; -// HighPerformanceTimer timer = new HighPerformanceTimer (); -// byte[] serData = null; - -// for (int i = 0; i < count; i++) -// bt.addRow (myGetRow (i)); - - -// ///serialization -// //create -// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out serData); -// deser.deserialize (new Stream [] {new MemoryStream (serData)}, out ops, out rows); -// ArrayList times = new ArrayList (); -// for (int i = 0; i < 25; i++) -// { -// timer.start (); -// ser.serialize (StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] {bt, bt}, out serData); -// timer.stop (); -// times.Add (timer.lastIntervalTime); -// Console.WriteLine (String.Format ("serialization of {0} rows, time = {1}, size = {2}", count, timer.lastIntervalTime, serData.Length)); -// } -// times.Sort (); -// Console.WriteLine ("Median time = {0}", times[times.Count/2]); -// //deserialization -// times.Clear (); -// for (int i = 0; i < 25; i++) -// { -// timer.start (); -// deser.deserialize (new Stream [] {new MemoryStream (serData)}, out ops, out rows); -// timer.stop(); -// times.Add (timer.lastIntervalTime); -// Console.WriteLine (String.Format ("deserialization of {0} rows, time = {1}", count, timer.lastIntervalTime)); -// } -// times.Sort (); -// Console.WriteLine ("Median time = {0}", times[times.Count/2]); -// } - -// private IBlueRow myGetRow (int id) -// { -// return new BlueRow (mySchema, -// Convert.ToByte (2), Convert.ToByte (4), DateTime.Now, id, 10.324555); -// } -// } -// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Test/Performance.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Test/Performance.cs deleted file mode 100644 index b1803fa04..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Test/Performance.cs +++ /dev/null @@ -1,46 +0,0 @@ -// using System; -// using NUnit.Framework; -// using BCG.Utilities.General; - -// namespace BCG -// { -// [TestFixture] -// public class Performance -// { -// public Performance() -// { -// } - -// [Test] -// public void testContructionTime () -// { -// double totalTime; -// string deserName = "BCG.ClientUniversalDeserializer"; -// HighPerformanceTimer timer = new HighPerformanceTimer (); -// IClientDeserializer cd; -// for (int i = 0; i < 100; i++) -// { -// timer.start (); -// cd = Activator.CreateInstance (Type.GetType (deserName)) as IClientDeserializer; -// timer.stop (); -// //Console.Write (timer.lastIntervalTime.ToString() + " "); -// } -// Console.WriteLine ("average time = " + timer.totalMilliseconds / 100); -// totalTime = timer.totalMilliseconds; - -// timer = new HighPerformanceTimer (); -// for (int i = 0; i < 100; i++) -// { -// timer.start (); -// cd = new ClientUniversalDeserializer (); -// timer.stop (); -// //Console.Write (timer.lastIntervalTime.ToString () + " "); -// } -// Console.WriteLine ("average time = " + timer.totalMilliseconds / 100); -// Console.WriteLine ("Ratio = " + totalTime/timer.totalMilliseconds); -// } - - - -// } -// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Test/TableAccumulatorTest.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Test/TableAccumulatorTest.cs deleted file mode 100644 index c2d7d79b3..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Test/TableAccumulatorTest.cs +++ /dev/null @@ -1,275 +0,0 @@ -// using System; -// using System.IO; -// using NUnit.Framework; - -// namespace BCG -// { -// [TestFixture] -// public class TableAccumulatorTest -// { -// private IBlueTableSchema mySchema = new BlueTableSchema ("test", -// new ColumnInfo ("id", 0, typeof(int)), -// new ColumnInfo ("x", 1, typeof(int))); -// private SubscriberCore mySubCore = new SubscriberCore (); -// private IBlueRowCollection myNr, myAr, myDr; - -// public TableAccumulatorTest() {} - -// [SetUp] -// public void tearDown () -// { -// myNr = myAr = myDr = null; -// } - -// #region create -// [Test] -// public void testCreate () -// { -// TableAccumulator ta = new TableAccumulator ("test", mySubCore); -// IBlueDataSerializer ser = new BlueDataSerializationController (); -// byte[] data; -// BlueTable bt = new BlueTable (mySchema); -// for (int i = 0; i < 100; i++) -// bt.addRow (new BlueRow (mySchema, i, 2*i)); -// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data); -// ta.onUpdate += new TableUpdateDelegate (myHandleUpdate); -// ta.update (new Stream[] {new MemoryStream (data)}); - -// //testing the data -// IBlueRowCollection dr, ar; -// //new rows -// BlueDataUtility.computeDifferences (bt, myNr, out dr, out ar); -// Assert.AreEqual (0, dr.Count); -// Assert.AreEqual (0, ar.Count); -// //added rows -// BlueDataUtility.computeDifferences (bt, myAr, out dr, out ar); -// Assert.AreEqual (0, dr.Count); -// Assert.AreEqual (0, ar.Count); -// //deleted rows; -// Assert.AreEqual (0, myDr.Count); -// ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate); -// } -// #endregion - -// #region single add, delete and change -// [Test] -// public void testAddRemoveChange () -// { -// TableAccumulator ta = new TableAccumulator ("test", mySubCore); -// IBlueDataSerializer ser = new BlueDataSerializationController (); -// byte[] data; -// BlueTable bt = new BlueTable (mySchema); -// for (int i = 0; i < 100; i++) -// bt.addRow (new BlueRow (mySchema, i, 2*i)); -// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data); -// ta.onUpdate += new TableUpdateDelegate (myHandleUpdate); -// ta.update (new Stream[] {new MemoryStream (data)}); - -// //add -// //add a row with id 100 -// bt.clear (); -// bt.addRow (new BlueRow (mySchema, 100, 200)); -// ser.serialize (StreamOpCode.add, new IBlueRowCollection[] {bt}, out data); -// ta.update (new Stream[] {new MemoryStream (data)}); -// //testing the add data -// Assert.AreEqual (101, myNr.Count); -// for (int i = 0; i < 101; i++) -// myNr.containsID (i); -// Assert.AreEqual (0, myDr.Count); -// Assert.AreEqual (1, myAr.Count); -// Assert.That (myAr.containsID (100)); - -// //remove -// //remove the row with id 0 -// bt.clear (); -// bt.addRow (new BlueRow (mySchema, 0, 0)); -// ser.serialize (StreamOpCode.remove, new IBlueRowCollection[] {bt}, out data); -// ta.update (new Stream[] {new MemoryStream (data)}); -// //testing remove data -// Assert.AreEqual (100, myNr.Count); -// for (int i = 0; i < 101; i++) -// { -// Assert.That ((i == 0 && !myNr.containsID (i)) || (i != 0 && myNr.containsID (i))); -// } -// Assert.AreEqual (0, myAr.Count); -// Assert.AreEqual (1, myDr.Count); -// Assert.That (myDr.containsID (0)); - -// //change -// bt.clear (); -// bt.addRow (new BlueRow (mySchema, 100, -100)); -// ser.serialize (StreamOpCode.change, new IBlueRowCollection[] {bt}, out data); -// ta.update (new Stream[] {new MemoryStream (data)}); -// //testing changed data -// Assert.AreEqual (100, myNr.Count); -// Assert.AreEqual (1, myAr.Count); -// Assert.AreEqual (100, BlueDataUtility.extractRow (myAr, 0)["id"]); -// Assert.AreEqual (-100, BlueDataUtility.extractRow (myAr, 0)["x"]); -// Assert.AreEqual (1, myDr.Count); -// Assert.AreEqual (100, BlueDataUtility.extractRow (myDr, 0)["id"]); -// Assert.AreEqual (200, BlueDataUtility.extractRow (myDr, 0)["x"]); -// ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate); -// } -// #endregion - -// #region multiple change -// public void testMultipleChange () -// { -// TableAccumulator ta = new TableAccumulator ("test", mySubCore); -// IBlueDataSerializer ser = new BlueDataSerializationController (); -// byte[] data; -// BlueTable bt = new BlueTable (mySchema); -// for (int i = 0; i < 100; i++) -// bt.addRow (new BlueRow (mySchema, i, 2*i)); -// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data); -// ta.onUpdate += new TableUpdateDelegate (myHandleUpdate); -// ta.update (new Stream[] {new MemoryStream (data)}); - -// //partial rows changed -// bt.clear (); -// for (int i = 0; i < 50; i++) -// bt.addRow (new BlueRow (mySchema, i, -2*i)); -// ser.serialize (StreamOpCode.multipleChange, new IBlueRowCollection[] {bt}, out data); -// ta.update (new Stream[] {new MemoryStream (data)}); -// //checking received data -// Assert.AreEqual (100, myNr.Count); -// foreach (IBlueRow r in myNr) -// { -// if ((int)r["id"] < 50) -// Assert.AreEqual ((int)r["id"] * -2, r["x"]); -// else -// Assert.AreEqual ((int)r["id"] * 2, r["x"]); -// } -// Assert.AreEqual (50, myAr.Count); -// foreach (IBlueRow r in myAr) -// Assert.AreEqual ((int)r["id"] * -2, r["x"]); -// Assert.AreEqual (50, myDr.Count); -// foreach (IBlueRow r in myDr) -// Assert.AreEqual ((int)r["id"] * 2, r["x"]); - -// //full rows changed -// bt.clear (); -// for (int i = 0; i < 100; i++) -// bt.addRow (new BlueRow (mySchema, i, -2*i)); -// ser.serialize (StreamOpCode.multipleChange, new IBlueRowCollection[] {bt}, out data); -// ta.update (new Stream[] {new MemoryStream (data)}); -// //checking received data -// Assert.AreEqual (100, myNr.Count); -// foreach (IBlueRow r in myNr) -// { -// Assert.AreEqual ((int)r["id"] * -2, r["x"]); -// } -// Assert.AreEqual (100, myAr.Count); -// foreach (IBlueRow r in myAr) -// Assert.AreEqual ((int)r["id"] * -2, r["x"]); -// Assert.AreEqual (100, myDr.Count); -// foreach (IBlueRow r in myDr) -// { -// if ((int)r["id"] < 50) -// Assert.AreEqual ((int)r["id"] * -2, r["x"]); -// else -// Assert.AreEqual ((int)r["id"] * 2, r["x"]); -// } -// ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate); -// } -// #endregion - -// #region multiple deletes/adds -// public void testMultipleDeletesAdds () -// { - -// TableAccumulator ta = new TableAccumulator ("test", mySubCore); -// IBlueDataSerializer ser = new BlueDataSerializationController (); -// byte[] data; -// BlueTable bt = new BlueTable (mySchema); -// for (int i = 0; i < 100; i++) -// bt.addRow (new BlueRow (mySchema, i, 2*i)); -// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data); -// ta.onUpdate += new TableUpdateDelegate (myHandleUpdate); -// ta.update (new Stream[] {new MemoryStream (data)}); - -// BlueTable at = new BlueTable (mySchema), -// dt = new BlueTable (mySchema); -// for (int i = 0; i < 50; i++) -// dt.addRow (new BlueRow (mySchema, i, 2*i)); -// for (int i = 100; i < 150; i++) -// at.addRow (new BlueRow (mySchema, i, 2*i)); -// ser.serialize (StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] {dt, at}, out data); -// ta.update (new Stream[] {new MemoryStream (data)}); - -// //checking state -// Assert.AreEqual (100, myNr.Count); -// for (int i = 50; i < 150; i++) -// Assert.That (myNr.containsID (i)); -// Assert.AreEqual (50, myDr.Count); -// for (int i = 0; i < 50; i++) -// Assert.That (myDr.containsID (i)); -// Assert.AreEqual (50, myAr.Count); -// for (int i = 100; i < 150; i++) -// Assert.That (myAr.containsID (i)); -// ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate); -// } -// #endregion - -// #region multiple ops in one stream -// [Test] -// public void testMultipleOps () -// { -// TableAccumulator ta = new TableAccumulator ("test", mySubCore); -// ta.onUpdate += new TableUpdateDelegate (myHandleUpdate); -// IBlueDataSerializer ser = new BlueDataSerializationController (); -// byte[] data; -// MemoryStream ms = new MemoryStream (); -// BlueTable bt = new BlueTable (mySchema); -// for (int i = 0; i < 100; i++) -// bt.addRow (new BlueRow (mySchema, i, 2*i)); -// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data); -// ta.update (new Stream[] {new MemoryStream (data)}); -// //ms.Write (data, 0, data.Length); -// //changing the first 5 rows -// bt.clear (); -// for (int i = 0; i < 5; i++) -// bt.addRow (new BlueRow (mySchema, i, 1000)); -// ser.serialize (StreamOpCode.multipleChange, new IBlueRowCollection[] {bt}, out data); -// ms.Write (data, 0, data.Length); -// //delete id 5 -// bt.clear (); -// bt.addRow (new BlueRow (mySchema, 5, 10)); -// ser.serialize (StreamOpCode.remove, new IBlueRowCollection[] {bt}, out data); -// ms.Write (data, 0, data.Length); -// //add id 100 -// bt.clear (); -// bt.addRow (new BlueRow (mySchema, 100, 200)); -// ser.serialize (StreamOpCode.add, new IBlueRowCollection[] {bt}, out data); -// ms.Write (data, 0, data.Length); -// //delete 4 and add 5 -// bt.clear (); -// BlueTable dels = new BlueTable (bt.schema); -// for (int i = 90; i < 94; i++) -// dels.addRow (new BlueRow (mySchema, i, i*2)); - -// for (int i = 101; i < 106; i++) -// bt.addRow (new BlueRow (mySchema, i, 2*i)); -// ser.serialize (StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] {dels, bt}, out data); -// ms.Write (data, 0, data.Length); -// ms.Position = 0; -// ta.update (new Stream[] {ms}); - -// Assert.AreEqual (101, myNr.Count); -// Assert.AreEqual (10, myDr.Count); -// Assert.AreEqual (11, myAr.Count); -// ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate); - -// } -// #endregion - -// private void myHandleUpdate (IBlueRowCollection nr, IBlueRowCollection ar, IBlueRowCollection dr) -// { -// myNr = nr; -// myAr = ar; -// myDr = dr; -// } - - -// } -// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Test/UniversalRowSerializerTest.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Test/UniversalRowSerializerTest.cs deleted file mode 100644 index e7882a50a..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Test/UniversalRowSerializerTest.cs +++ /dev/null @@ -1,58 +0,0 @@ -// using System; -// using System.IO; -// using NUnit.Framework; - -// namespace BCG -// { -// [TestFixture] -// public class UniversalRowSerializerTest -// { -// private IBlueRowSerializer myRowSerializer; -// private IClientRowDeserializer myRowDeserializer; -// private Random myRnd = new Random (DateTime.Now.Millisecond); -// private IBlueTableSchema mySchema = new BlueTableSchema ("Test", -// new ColumnInfo ("id", 0, typeof(int)), -// new ColumnInfo ("int1", 1, typeof(int)), -// new ColumnInfo ("int2", 2, typeof(int)), -// new ColumnInfo ("double1", 3, typeof(double)), -// new ColumnInfo ("double2", 4, typeof(double)), -// new ColumnInfo ("byte1", 5, typeof(byte)), -// new ColumnInfo ("string1", 6, typeof(string)), -// new ColumnInfo ("string2", 7, typeof(string))); -// public UniversalRowSerializerTest () {} - -// [Test] -// public void testSerialiazationCycle () -// { -// myRowSerializer = new UniversalRowSerializer (mySchema); -// myRowDeserializer = new ClientUniversalRowDeserializer (mySchema); -// for (int i = 0; i < 10000; i++) -// { -// BinaryWriter bw = new BinaryWriter (new MemoryStream ()); -// IBlueRow ir = myGetRandomRow (); -// myRowSerializer.serializeRow (ir, StreamOpCode.add, bw); - -// IBlueRow or; -// (bw.BaseStream as MemoryStream).Position = 0; -// BinaryReader br = new BinaryReader (bw.BaseStream); -// myRowDeserializer.deserializeRow (null, br, StreamOpCode.add, out or); -// Assert.That (BlueDataUtility.computeRowEquality (ir, or)); -// } -// } - -// private IBlueRow myGetRandomRow () -// { -// return new BlueRow (mySchema, -// myRnd.Next (1000), -// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : myRnd.Next (1000) as object, -// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object: myRnd.Next (1000) as object, -// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : myRnd.NextDouble () as object, -// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : myRnd.NextDouble () as object, -// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : Convert.ToByte (76) as object, -// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : "abcdefgh" as object, -// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : "1234567890" as object -// ); - -// } -// } -// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs index 7d3f35b3d..1965d8b57 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs @@ -81,6 +81,13 @@ namespace bts { serialize_column_info(ci, ref bw); } + public static byte[] serialize_schema(IBlueTableSchema schema) { + MemoryStream str = new MemoryStream(); + BinaryWriter bw = new BinaryWriter(str); + serialize_schema(schema, ref bw); + return str.ToArray(); + } + public static IBlueTableSchema deserialize_schema(ref BinaryReader br) { var hex_txt = to_hex(br.BaseStream); bts.Log.Gen.info($"StreamSerializationUtils.deserialize_schema bytes: {hex_txt}"); @@ -94,6 +101,11 @@ namespace bts { return new BlueTableSchema(table_name, cis.ToArray()); } + public static IBlueTableSchema deserialize_schema(byte[] ser_bytes) { + BinaryReader br = new BinaryReader(new MemoryStream(ser_bytes)); + return deserialize_schema(ref br); + } + public static void serialize_column_info(ColumnInfo ci, ref BinaryWriter bw) { // bw.Write(ci.colNo); @@ -162,6 +174,18 @@ namespace bts { return str.ToArray(); } + public static byte[] serialize_row(this IStreamRowSerializer ser, IBlueRow row) { + MemoryStream str = new MemoryStream(); + BinaryWriter bw = new BinaryWriter(str); + ser.serialize_row(row, StreamIOMessage.StreamOp.CREATE, ref bw); + return str.ToArray(); + } + + public static IBlueRow deserialize_row(this IStreamRowDeserializer row_deser, byte[] row_bytes) { + BinaryReader br = new BinaryReader(new MemoryStream(row_bytes)); + return row_deser.deserialize_row(ref br); + } + public static StreamIOMessage deserialize(this bts.IStreamMsgDeserilizer deser, byte[] ser_msg) { MemoryStream str = new MemoryStream(ser_msg); BinaryReader br = new BinaryReader(str);