TimeSeries and TimeBucketStreams, Stream offloading etc

This commit is contained in:
Taha committed 2023-10-23 14:18:31 -04:00
1 parent 33030b1212
commit 88bf04e5f1
31 files changed
+1823 -836

No files matched your search

@@ -3,16 +3,15 @@ using System.IO;
using System.Collections;
using BCG.Utilities;
namespace BCG
{
namespace BCG {
/// <summary>
/// Controls (ie creates dynamically) an IClientDeserializer and then passes data to it.
/// </summary>
public class ClientDeserializationController : IClientDeserializer
{
public class ClientDeserializationController : IClientDeserializer {
private IClientDeserializer myDeserializer;
public ClientDeserializationController () {}
private int _n_desers = 0;
public ClientDeserializationController() { }
#region IClientDeserializer Members
@@ -23,42 +22,41 @@ namespace BCG
/// <param name="streams"></param>
/// <param name="theOps"></param>
/// <param name="theRows"></param>
public void deserialize (Stream[] streams, out byte[] theOps, out Array theRows)
{
if (myDeserializer != null)
{
myDeserializer.deserialize (streams, out theOps, out theRows);
}
else
{
Stream totalStream = UtilityMethods.getAggregatedStream (streams);
public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) {
bts.Log.Gen.info($"ClientDeserializationController.deserialize len: {streams[0].Length - streams[0].Position}");
if (myDeserializer != null) {
myDeserializer.deserialize(streams, out theOps, out theRows);
} else {
Stream totalStream = UtilityMethods.getAggregatedStream(streams);
long initialPos = totalStream.Position;
BinaryReader br = new BinaryReader (totalStream);
BinaryReader br = new BinaryReader(totalStream);
byte op;
try
{
op = br.ReadByte ();
}
catch (EndOfStreamException)
{
try {
op = br.ReadByte();
} catch (EndOfStreamException) {
op = Byte.MaxValue;
}
if (op == StreamOpCode.create)
{
br.ReadString (); //pubsubclientdeserializer
string deserName = br.ReadString ();
myDeserializer = Activator.CreateInstance (Type.GetType (deserName))
if (op == StreamOpCode.create) {
br.ReadString(); //pubsubclientdeserializer
string deserName = br.ReadString();
myDeserializer = Activator.CreateInstance(Type.GetType(deserName))
as IClientDeserializer;
totalStream.Position = initialPos;
myDeserializer.deserialize (new Stream[] {totalStream}, out theOps, out theRows);
}
else
{
myDeserializer.deserialize(new Stream[] { totalStream }, out theOps, out theRows);
} else {
// BlueEventLog.logEvent ("Received a non create before a create", BlueEventLevel.Warning);
theOps = new byte[]{};
theRows = new object[]{};
theOps = new byte[] { };
theRows = new object[] { };
}
}
var msgs = StreamIOMessage.assemble_msgs(theOps, theRows);
int i = 0;
foreach (var m in msgs) {
bts.Log.Gen.info($"ClientDeserializationController n_desers: {_n_desers} msg no: {i++}: {m}");
}
_n_desers++;
// log it here
}
@@ -5,61 +5,61 @@ using System.IO;
using BCG.Utilities;
//using BCG.Utilities.BlueConfiguration;
namespace BCG
{
namespace BCG {
/// <summary>
/// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process
/// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations)
/// </summary>
internal class ClientUniversalRowDeserializer : IClientRowDeserializer
{
internal class ClientUniversalRowDeserializer : IClientRowDeserializer {
private IBlueTableSchema mySchema;
private int myNullMapLength = 0;
private PrimitiveDeserializationHelper myDeserializationHelper;
private BitArray myNullMap;
private int[] mySerIndexToSchemaIndexMap;
public ClientUniversalRowDeserializer (IBlueTableSchema schema)
{
myDeserializationHelper = new PrimitiveDeserializationHelper (mySchema = schema);
myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8)));
myInitReorderSchema ();
public ClientUniversalRowDeserializer(IBlueTableSchema schema) {
myDeserializationHelper = new PrimitiveDeserializationHelper(mySchema = schema);
myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length) / 8)));
myInitReorderSchema();
}
#region Schema Reordering initialization
private void myInitReorderSchema ()
{
int idColIndex = mySchema["id"].colNo;
int[] temp = new int [mySchema.columnInfo.Length];
foreach (ColumnInfo ci in mySchema.columnInfo)
{
temp[ci.colNo] =
ci.colNo == idColIndex ? 0 :
ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo;
private void myInitReorderSchema() {
try {
int idColIndex = mySchema["id"].colNo;
int[] temp = new int[mySchema.columnInfo.Length];
foreach (ColumnInfo ci in mySchema.columnInfo) {
temp[ci.colNo] =
ci.colNo == idColIndex ? 0 :
ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo;
}
mySerIndexToSchemaIndexMap = new int[temp.Length];
for (int i = 0; i < temp.Length; i++)
mySerIndexToSchemaIndexMap[temp[i]] = i;
} catch (Exception ex) {
string schema_name = mySchema == null ? "NULL" : mySchema.tableName;
string err_msg = $"ClientUniversalRowDeserializer.myInitReorderSchema table: {schema_name} ex: {ex}";
bts.Log.Gen.error(err_msg);
throw new ApplicationException(err_msg);
}
mySerIndexToSchemaIndexMap = new int[temp.Length];
for (int i = 0; i < temp.Length; i ++)
mySerIndexToSchemaIndexMap [temp[i]] = i;
}
#endregion
#region IClientRowDeserializer Members
public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
{
br.ReadInt32 (); //length information for the row not needed
myNullMap = new BitArray(br.ReadBytes (myNullMapLength));
object[] values = new object [mySchema.columnNames.Length];
for (int i = 0; i < values.Length; i++)
{
public void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) {
br.ReadInt32(); //length information for the row not needed
myNullMap = new BitArray(br.ReadBytes(myNullMapLength));
object[] values = new object[mySchema.columnNames.Length];
for (int i = 0; i < values.Length; i++) {
int colNo = mySerIndexToSchemaIndexMap[i];
values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn (colNo, br);
values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn(colNo, br);
}
r = new BlueRow(mySchema, values);
r = new BlueRow(mySchema, values);
}
@@ -1,22 +1,149 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace BCG {
public class StreamIOMessage {
public enum StreamOp {
Create = 0,
Add = 1, //adds a single row
Remove = 2, //removes a single row
Change = 3, //changes a single row
MultipleRemoveAdd = 4,//removes and adds multiple rows
MultipleChange = 5 //changes multiples rows
}
public class CreateMessage {
public IBlueRowCollection rows;
public override string ToString() {
string rv = "Create message ";
if (rows == null) {
rv += " rows: null";
return rv;
}
foreach (IBlueRow r in rows) {
rv += $"id: {r["id"]} ";
}
return rv;
}
}
public class AddMessage {
public IBlueRow added_row;
}
public class RemoveMessage {
public IBlueRow removed_row;
}
public class ChangeMessage {
public IBlueRow changed_row;
}
public class MultipleRemoveAddMessage {
public IBlueRowCollection removed;
public IBlueRowCollection added;
public override string ToString() {
string rv = "MultipleRemoveAdd removed: ";
foreach (IBlueRow r in removed) {
rv += $"id: {r["id"]} ";
}
rv += "added: ";
foreach (IBlueRow r in added) {
rv += $"id: {r["id"]} ";
}
return rv;
}
}
public class MultipleChangeMessage {
public IBlueRowCollection changed;
}
public StreamOp op;
public CreateMessage create_message = null;
public AddMessage add_message = null;
public RemoveMessage remove_message = null;
public ChangeMessage change_message = null;
public MultipleRemoveAddMessage multiple_remove_add_message = null;
public MultipleChangeMessage multiple_change_message = null;
public StreamIOMessage(byte op_, object data) {
op = (StreamIOMessage.StreamOp)(op_);
switch (op) {
case StreamIOMessage.StreamOp.Create:
IBlueRowCollection state = null;
// TODO, not sure why it can be an array and by itself
if (data is IBlueRowCollection) state = data as IBlueRowCollection;
else if (data is IBlueRowCollection[]) state = (data as IBlueRowCollection[]).ElementAt(0);
create_message = new CreateMessage() { rows = state };
break;
case StreamIOMessage.StreamOp.Add:
bts.Log.assert(false);
break;
case StreamIOMessage.StreamOp.Remove:
bts.Log.assert(false);
break;
case StreamIOMessage.StreamOp.Change:
bts.Log.assert(false);
break;
case StreamIOMessage.StreamOp.MultipleRemoveAdd:
IBlueRowCollection[] row_args = data as IBlueRowCollection[];
multiple_remove_add_message = new MultipleRemoveAddMessage() { removed = row_args[0], added = row_args[1] };
break;
case StreamIOMessage.StreamOp.MultipleChange:
//IBlueRowCollection[] row_args = data as IBlueRowCollection[];
multiple_change_message = new MultipleChangeMessage() { changed = (data as IBlueRowCollection[])[0] };
break;
}
}
public override string ToString() {
switch (op) {
case StreamIOMessage.StreamOp.Create:
return create_message.ToString();
break;
case StreamIOMessage.StreamOp.Add:
bts.Log.assert(false);
return String.Empty;
break;
case StreamIOMessage.StreamOp.Remove:
bts.Log.assert(false);
return String.Empty;
break;
case StreamIOMessage.StreamOp.Change:
bts.Log.assert(false);
return String.Empty;
break;
case StreamIOMessage.StreamOp.MultipleRemoveAdd:
return multiple_remove_add_message.ToString();
break;
case StreamIOMessage.StreamOp.MultipleChange:
return multiple_change_message.ToString();
break;
}
bts.Log.assert(false);
return String.Empty;
}
public static IEnumerable<StreamIOMessage> assemble_msgs(byte[] ops, Array op_data) {
int i = 0;
List<StreamIOMessage> rv = new List<StreamIOMessage>();
foreach (byte op in ops) {
object data = op_data.GetValue(i++);
rv.Add(new StreamIOMessage(op, data));
}
return rv;
}
}
namespace BCG
{
/// <summary>
/// Performs deserialization inside the client app. The client gets either the complete data set or the
/// increments from PubSub and this interface makes sense out of them from a tabular/datastream stand point.
/// </summary>
public interface IClientDeserializer
{
public interface IClientDeserializer {
/// <summary>
/// deserializes the data received by the client from the pubsub
/// </summary>
/// <param name="serializedData">the data in byte format received from pubsub</param>
/// <param name="theOps">the operations that the data maps onto</param>
/// <param name="theRows">the data corresponding to the operations, is an array of IblueRowCollection[]</param>
void deserialize (Stream[] streams, out byte[] theOps, out Array theRows);
void deserialize(Stream[] streams, out byte[] theOps, out Array theRows);
}
}
@@ -2,13 +2,11 @@ using System;
using System.IO;
using System.Collections;
namespace BCG
{
namespace BCG {
/// <summary>
/// Deserializes one row. the current state is also provided as some deserialization schemes might use it.
/// </summary>
public interface IClientRowDeserializer
{
void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r);
public interface IClientRowDeserializer {
void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r);
}
}