Adding TableMetaDataStream

This commit is contained in:
Taha authored and Adeel Khan committed 2024-10-03 17:55:30 +05:00
1 parent 4b76442a8f
commit d2a2d5fb37
4 files changed
+80 -51

No files matched your search

@@ -24,7 +24,6 @@ namespace bts {
var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
myFireEvent(dp_out);
}
@@ -5,18 +5,21 @@ using System.Linq;
using bts;
namespace BCG {
namespace BCG
{
// This class is responsible for mantaining the state fo a table. It gets the
// bytes from the pubsubserver via the handle_updates call (called by VSD), deserializes
// updates the state of the table, and fires the TableUpdateDelegate.
public class TableAccumulator2 : ITableAccumulator {
public class TableAccumulator2 : ITableAccumulator
{
private readonly string _tag;
private readonly ISubscriberCore _subscriber_core;
IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
private BlueTable _table;
private TableUpdateDelegate _on_update_delegate;
public TableAccumulator2(string tag, ISubscriberCore sub_core) {
public TableAccumulator2(string tag, ISubscriberCore sub_core)
{
_tag = tag;
_subscriber_core = sub_core;
}
@@ -25,10 +28,13 @@ namespace BCG {
//public bool is_active() { return _on_update_delegate != null; }
public event TableUpdateDelegate on_update {
add {
public event TableUpdateDelegate on_update
{
add
{
Log.Gen.info("TableAccumulator2.on_update.add {_tag} subscribed");
if (_on_update_delegate == null) {
if (_on_update_delegate == null)
{
//ie first subscription
Log.Gen.info("TableAccumulator2.on_update {_tag} subscribed for the first time");
_subscriber_core.subscribe(_tag);
@@ -38,14 +44,17 @@ namespace BCG {
if (_table != null)
value(_table, _table, new BlueTable(_table.schema));
}
remove {
remove
{
Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed");
if (_on_update_delegate != null) {
if (_on_update_delegate != null)
{
_on_update_delegate =
Delegate.Remove(_on_update_delegate, value) as TableUpdateDelegate;
int n_subs = _on_update_delegate == null ? 0 : _on_update_delegate.GetInvocationList().Length;
Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed n_subs: {n_subs}");
if (_on_update_delegate == null) {
if (_on_update_delegate == null)
{
_subscriber_core.unsubscribe(_tag);
// TODO
//_table = null;
@@ -55,16 +64,19 @@ namespace BCG {
}
}
private int _update_no = 0;
public void handle_updates(IEnumerable<byte[]> msgs) {
public void handle_updates(IEnumerable<byte[]> msgs)
{
if (msgs == null || msgs.Count() == 0)
return;
Log.Gen.info($"TableAccumulator2.handle_updates _update_no: {_update_no++} msgs: {msgs.Count()} msg_0: {msgs.ElementAt(0).Length}");
IBlueRowCollection table_0 = _table == null ? null : new BlueTable(_table);
foreach (var msg in msgs) {
foreach (var msg in msgs)
{
bts.StreamIOMessage str_msg = _deser.deserialize(msg);
var state_txt = _table == null ? "NULL" : BlueDataUtility.id_string(_table);
Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} sz: {msg.Length} str_msg: {str_msg} _state: {state_txt}");
switch (str_msg.op) {
switch (str_msg.op)
{
case bts.StreamIOMessage.StreamOp.CREATE:
_table = new BlueTable(str_msg.create_message.rows);
break;
@@ -77,11 +89,13 @@ namespace BCG {
// this could be made faster if we just do this on the CREATE message, and pass-through
// what we get on the MULTIPLE_REMOVE_ADD
IBlueRowCollection d_rows, a_rows;
BlueDataUtility.compute_differences_id_based(_table, table_0, out d_rows, out a_rows);
//BlueDataUtility.compute_differences_id_based(_table, table_0, out d_rows, out a_rows);
BlueDataUtility.compute_differences_fast(_table, table_0, out d_rows, out a_rows);
_on_update_delegate(_table, a_rows, d_rows);
}
public void update(Stream[] streams) {
public void update(Stream[] streams)
{
}
#endregion
@@ -1,6 +1,6 @@
{
"user": "bts",
"sparkd": "http://localhost:8000/config",
"sparkd": "http://192.168.1.80:8000/config",
"pubsub": {
"milliseconds_between_sub_info_updates": 1000,
"milliseconds_between_time_slice_updates": 5000,
@@ -18,14 +18,14 @@
"publisher_routes": [
{
"name": "default",
"server_address": "bluto://localhost:12345",
"server_address": "bts-cloud://127.0.0.1:12345",
"tag": ".*"
}
],
"subscriber_routes": [
{
"name": "default",
"server_address": "bluto://localhost:12345",
"server_address": "bts-cloud://127.0.0.1:12345",
"tag": ".*"
}
],
@@ -106,4 +106,4 @@
]
}
]
}
}
@@ -1,23 +1,27 @@
namespace bts {
namespace bts
{
using BCG;
using BCG.Utilities.General;
public class TableMetaDataStream : ViewStream, IWorker {
private class TableState {
public class TableMetaDataStream : ViewStream, IWorker
{
private class TableState
{
public ViewStream table_stream;
public IBlueTableSchema schema = null;
public bool processed = false;
public TableState(ViewStream table_stream_) {
public TableState(ViewStream table_stream_)
{
table_stream = table_stream_;
table_stream.onChange += _handle_table_update;
}
private void _handle_table_update(object dp_obj) {
private void _handle_table_update(object dp_obj)
{
DataPacket dp = dp_obj as DataPacket;
schema = dp.schema;
}
public void unsub() {
public void unsub()
{
table_stream.onChange -= _handle_table_update;
}
}
@@ -34,62 +38,74 @@ namespace bts {
private BlueTable _state;
public TableMetaDataStream()
: this(ViewStreamDatabase.instance) { }
public TableMetaDataStream(ViewStreamDatabase vsd) {
public TableMetaDataStream(ViewStreamDatabase vsd)
{
_vsd = vsd;
_base_stream = new PubSubServerStateStream(_vsd);
_state = new BlueTable(_schema);
}
protected override void myOnFirstSubscriber() {
protected override void myOnFirstSubscriber()
{
_base_stream.onChange += _handle_base_stream;
_vsd.addWorker(this);
}
protected override void myOnLastSubscriber() {
protected override void myOnLastSubscriber()
{
_base_stream.onChange -= _handle_base_stream;
_vsd.removeWorker(this);
}
public void doWork() {
public void doWork()
{
bool changed = false;
foreach (TableState ts in _table_map.Values) {
if (!ts.processed) {
if (ts.schema != null) {
foreach (ColumnInfo ci in ts.schema.columnInfo) {
int id = _state.Count;
var table_name = ts.schema.tableName;
var col_idx = ci.colNo;
var col_name = ci.colName;
TypeCode type_code = BlueDataUtility.get_typecode(ci.colType);
BlueRow nr = new BlueRow(_schema, id, table_name, col_idx, Convert.ToInt32(type_code),
col_name);
changed = true;
_state.addRow(nr);
}
foreach (TableState ts in _table_map.Values)
{
if (!ts.processed && ts.schema != null)
{
foreach (ColumnInfo ci in ts.schema.columnInfo)
{
int id = _state.Count;
var table_name = ts.schema.tableName;
var col_idx = ci.colNo;
var col_name = ci.colName;
TypeCode type_code = BlueDataUtility.get_typecode(ci.colType);
BlueRow nr = new BlueRow(_schema, id, table_name, col_idx, Convert.ToInt32(type_code),
col_name);
changed = true;
_state.addRow(nr);
}
ts.processed = true;
ts.unsub();
}
}
if (changed) {
if (changed)
{
var dp_out = BlueDataUtility.build_dp_inc(_state, myLastValue as DataPacket);
myFireEvent(dp_out);
}
}
private void _handle_base_stream(object dp_obj) {
private void _handle_base_stream(object dp_obj)
{
DataPacket dp = dp_obj as DataPacket;
foreach (IBlueRow r in dp.newRows) {
foreach (IBlueRow r in dp.newRows)
{
string tag = r.get<string>("tag");
if (tag.StartsWith("DS_")) {
if (tag.StartsWith("DS_"))
{
_handle_table(tag);
}
}
}
private void _handle_table(string tag_name) {
private void _handle_table(string tag_name)
{
Log.assert(tag_name.StartsWith("DS_"));
string table_name = tag_name.Remove(0, 3);
TableState ts = null;
if (!_table_map.TryGetValue(table_name, out ts)) {
if (!_table_map.TryGetValue(table_name, out ts))
{
ts = new TableState(new TableStream(table_name, _vsd));
_table_map.Add(table_name, ts);
}