Cleaning Config 1.0 (not completely done)

This commit is contained in:
Taha Afzal authored and Adeel Khan committed 2024-10-03 17:55:30 +05:00
1 parent 09b863f2ab
commit 4b76442a8f
6 files changed
+181 -167

No files matched your search

@@ -1,6 +1,54 @@
using System;
using BCG.Utilities.General;
using bts;
namespace BCG {
[DataStreamUserRespresentation("milliseconds")]
public class MillisecondStream : ViewStream, IWorker {
private DateTime _last_fire_time = DateTime.MinValue;
private readonly TimeSpan _period;
private readonly int _size;
private readonly ViewStreamDatabase _vsd;
private IBlueTableSchema _schema;
private bts.CircularBufferTable _table;
public MillisecondStream(int interval_msecs, int size)
: this(ViewStreamDatabase.instance, interval_msecs, size) {
}
public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
_vsd = vsd;
_period = TimeSpan.FromMilliseconds(interval_msecs);
_size = size;
_schema = BlueDataUtility.form_schema("Milliseconds",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("time", TypeCode.DateTime),
Tuple.Create("display_time", TypeCode.String));
_table = new CircularBufferTable(_schema, _size);
}
protected override void myOnFirstSubscriber() {
_vsd.addWorker(this);
}
protected override void myOnLastSubscriber() {
_vsd.removeWorker(this);
}
private int _id = 0;
public void doWork() {
if (DateTime.Now - _last_fire_time >= _period) {
_last_fire_time = DateTime.Now;
IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
_last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
_table.add_row(nr);
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
myFireEvent(dp);
}
}
}
public class MSecondStream : TimeUnitStream {
public MSecondStream(int intervalInms, int size)
: base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
@@ -16,12 +16,6 @@ namespace bts {
_buffer_sz = buffer_sz;
}
// public OrderedBlueTable(List<IBlueRow> rows, int buffer_sz) {
// _buffer_sz = buffer_sz;
// foreach (IBlueRow r in rows)
// add_row(r);
// }
// returns the row that might need to be deleted to keep
// the circular buffer intact
public IBlueRow add_row(IBlueRow row) {
@@ -38,10 +38,6 @@ public class DataStreamPublisherConfig {
[JsonPropertyName("publishers")]
public List<PublisherNode>? Publishers { get; set; }
[JsonPropertyName("simple_echo_client")]
[Option("simple-echo-client", HelpText = "Run a simple echo client (won't publish tables or connect to sparkd)")]
public bool? SimpleEchoClient { get; set; }
[JsonPropertyName("pubsub")]
public PubSubConfig? PubSub { get; set; }
@@ -54,7 +50,6 @@ public class DataStreamPublisherConfig {
public static DataStreamPublisherConfig Default() =>
new() {
SimpleEchoClient = false,
PubSub = PubSubConfig.Default(),
};
@@ -64,7 +59,6 @@ public class DataStreamPublisherConfig {
Sparkd ??= other.Sparkd;
Publishers ??= other.Publishers;
PubSub ??= other.PubSub;
SimpleEchoClient ??= other.SimpleEchoClient;
ConfigFile ??= other.ConfigFile;
}
@@ -23,10 +23,8 @@ using System.Threading.Tasks;
await Parser.Default.ParseArguments<DataStreamPublisherConfig>(args)
.MapResult(RunAsync, errs => Task.FromResult(1));
async Task<int> RunAsync(DataStreamPublisherConfig cfg)
{
try
{
async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
try {
cfg.MergeWithConfigFile();
cfg.MergeWith(DataStreamPublisherConfig.Default());
PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub));
@@ -34,17 +32,8 @@ async Task<int> RunAsync(DataStreamPublisherConfig cfg)
Log.Gen.info("Starting DataStreamPublisher");
var service_ctx = ServiceContext.create(cfg.Sparkd);
await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty);
if (cfg.SimpleEchoClient ?? false)
{
start_echo_client();
}
else
{
_publish_streams(cfg.Publishers ?? new(0));
}
}
catch (Exception ex)
{
_publish_streams(cfg.Publishers ?? new(0));
} catch (Exception ex) {
// Write the error to console first, in case the logging system is compromised.
var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
Console.Error.WriteLine(error_msg);
@@ -55,51 +44,29 @@ async Task<int> RunAsync(DataStreamPublisherConfig cfg)
return 0;
}
static ViewStream? build_stream(string? stream_expression)
{
if (stream_expression is null)
{
static ViewStream? build_stream(string? stream_expression) {
if (stream_expression is null) {
return null;
}
try
{
try {
bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv;
}
catch (Exception ex)
{
} catch (Exception ex) {
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
return null;
}
}
static void start_echo_client()
{
Console.WriteLine("Starting DSPublisher in TCP client echo mode, Hit key to continue");
Console.ReadLine();
SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345, 1000, 1000, Int32.MaxValue);
client.start();
while (true)
{
System.Threading.Thread.Sleep(1000);
}
// await SimpleTCPClient.start("127.0.0.1", 12345);
// Task.Delay(1000000000).Wait();
}
void _publish_streams(IEnumerable<PublisherNode> publishers)
{
void _publish_streams(IEnumerable<PublisherNode> publishers) {
PubTableBroker pb_broker = new PubTableBroker();
RemoteQueryManager csm = new RemoteQueryManager();
var csvPublishers = new List<CSVPublisher>();
foreach (var publisher in publishers)
{
if (Directory.Exists(publisher.CsvDir))
{
foreach (var publisher in publishers) {
if (Directory.Exists(publisher.CsvDir)) {
csvPublishers.Add(new(publisher.CsvDir, pb_broker));
}
}
@@ -108,8 +75,7 @@ void _publish_streams(IEnumerable<PublisherNode> publishers)
.SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher))
.ToList();
while (true)
{
while (true) {
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
ViewStreamDatabase.instance.doWork();
csm.doWork();
@@ -117,19 +83,16 @@ void _publish_streams(IEnumerable<PublisherNode> publishers)
ManagedSubscriber.instance.doWork();
}
ViewPublisher? createViewPublisher(PublishedStream ps)
{
ViewPublisher? createViewPublisher(PublishedStream ps) {
Log.Gen.info(
$"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
if (string.IsNullOrEmpty(ps.Name))
{
if (string.IsNullOrEmpty(ps.Name)) {
Log.Gen.warn(
$"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
return null;
}
var vs = build_stream(ps.Expression);
if (vs is null)
{
if (vs is null) {
Log.Gen.warn(
$"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
return null;
@@ -140,39 +103,31 @@ void _publish_streams(IEnumerable<PublisherNode> publishers)
}
}
void on_change(object obj)
{
void on_change(object obj) {
bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
}
public class CSVPublisher : IWorker
{
public class CSVPublisher : IWorker {
private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
private readonly string _csv_directory;
private PubTableBroker _pub_table_broker;
public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker)
{
public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
_csv_directory = csv_directory;
_pub_table_broker = pub_table_broker;
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
}
public void doWork()
{
public void doWork() {
var file_names = Directory.GetFiles(_csv_directory);
foreach (var fn in file_names)
{
if (!_pubs.ContainsKey(fn))
{
foreach (var fn in file_names) {
if (!_pubs.ContainsKey(fn)) {
_handle_new_file(fn);
}
}
}
private void _handle_new_file(string file_name)
{
if (file_name.EndsWith(".csv"))
{
private void _handle_new_file(string file_name) {
if (file_name.EndsWith(".csv")) {
ViewStream vs = new CSVReaderStream(file_name);
vs = new SchemaGuesserStream(vs);
var toks = file_name.Split('\\');
@@ -27,35 +27,39 @@ public class PubSubConfig {
[JsonPropertyName("simple_echo_server")]
public bool? SimpleEchoServer { get; set; }
// unused
[JsonPropertyName("milliseconds_between_sub_info_updates")]
public int? MillisecondsBetweenSubInfoUpdates { get; set; } = 1000;
// unused
[JsonPropertyName("milliseconds_between_time_slice_updates")]
public int? MillisecondsBetweenTimeSliceUpdates { get; set; } = 5000;
// unused
[JsonPropertyName("milliseconds_between_no_buffer_space_retries")]
public int? MillisecondsBetweenNoBufferRetries { get; set; } = 100;
// unused
[JsonPropertyName("retries_on_no_buffer_space")]
public int? RetriesOnNoBufferSpace { get; set; } = 1;
// unused
[JsonPropertyName("do_it_all_seconds_between_log")]
public int? DoItAllSecondsBetweenLog { get; set; } = 0;
// Milliseconds
// Used by the PubSubMessageReactor to pass to the ManagedTCPMessageHb (but unsed within it)
[JsonPropertyName("heartbeat_interval")]
public int? HeartbeatInterval { get; set; } = 15000;
[JsonPropertyName("missing_heartbeats_before_gc")]
public int? MissingHeartbeatsBeforeGc { get; set; } = 50;
[JsonPropertyName("use_managed_sockets")]
public bool? UseManagedSockets { get; set; } = true;
// unused
// Milliseconds
[JsonPropertyName("state_update_period")]
public int? StateUpdatePeriod { get; set; } = 5000;
// unused
// Milliseconds
[JsonPropertyName("statistics_update_period")]
public int? StatisticsUpdatePeriod { get; set; } = 5000;
@@ -64,67 +68,20 @@ public class PubSubConfig {
[JsonPropertyName("disposed_connection_check_period")]
public int? DisposedConnectionCheckPeriod { get; set; } = 1000;
// Milliseconds
[JsonPropertyName("message_processing_warning_threshold")]
public int? MessageProcessingWarningThreshold { get; set; } = 1000;
[JsonPropertyName("max_packet_size")]
public int? MaxPacketSize { get; set; } = 500000000;
[JsonPropertyName("blue_comm_must_die")]
public bool? BlueCommMustDie { get; set; } = true;
[JsonPropertyName("crushable_tags")]
public List<string>? CrushableTags { get; set; }
[JsonPropertyName("tag_type_to_assembly_map")]
public TagTypeToAssembly? TagTypeToAssemblyMap { get; set; }
[JsonPropertyName("server_queue")]
public string? ServerQueue { get; set; } = "PubSubQueue";
[JsonPropertyName("port_range")]
public PubSubPortRange? PortRange { get; set; }
[JsonPropertyName("bind_to_random_port")]
public bool? BindToRandomPort { get; set; } = false;
[JsonPropertyName("debug_tags")]
public string? DebugTags { get; set; }
// unused
[JsonPropertyName("latency_report_threshold_ms")]
public int? LatencyReportThresholdMs { get; set; }
[JsonPropertyName("use_substations")]
public bool? UseSubstations { get; set; }
[JsonPropertyName("use_dynamic_clients")]
public bool? UseDynamicClients { get; set; }
[JsonPropertyName("is_substation")]
public bool? IsSubstation { get; set; }
// unused
[JsonPropertyName("internally_publish_sub_info")]
public bool? InternallyPublishSubInfo { get; set; }
[JsonPropertyName("use_managed_clients_in_native_code")]
public bool? UseManagedClientsInNativeCode { get; set; }
[JsonPropertyName("compressed_tags")]
public string? CompressedTags { get; set; }
[JsonPropertyName("tags_to_report_compression_ratio")]
public string? TagsToReportCompressionRatio { get; set; }
[JsonPropertyName("disable_msmq")]
public bool? DisableMsmq { get; set; }
[JsonPropertyName("use_prepublished_server_state")]
public bool? UsePrepublishedServerState { get; set; }
[JsonPropertyName("ip_translation")]
public string? IpTranslation { get; set; }
[JsonPropertyName("gc_activation_threshold")]
public int? GcActivationThreshold { get; set; }
@@ -148,38 +105,22 @@ public class PubSubConfig {
DoItAllSecondsBetweenLog = 0,
HeartbeatInterval = 15000,
MissingHeartbeatsBeforeGc = 50,
UseManagedSockets = true,
StateUpdatePeriod = 5000,
StatisticsUpdatePeriod = 5000,
DisposedConnectionCheckPeriod = 1000,
MessageProcessingWarningThreshold = 1000,
ServerPort = 12345,
MaxPacketSize = 500000000,
BlueCommMustDie = true,
CrushableTags = new(0),
TagTypeToAssemblyMap = new(),
ServerQueue = "PubSubQueue",
PortRange = new(30400, 31700),
BindToRandomPort = false,
DebugTags = NOTAGS,
LatencyReportThresholdMs = 1500,
UseSubstations = false,
IsSubstation = false,
InternallyPublishSubInfo = true,
UseManagedClientsInNativeCode = false,
CompressedTags = NOTAGS,
TagsToReportCompressionRatio = NOTAGS,
DisableMsmq = false,
UsePrepublishedServerState = false,
IpTranslation = null,
GcActivationThreshold = 0,
WaitPeriodAfterGc = 1000,
SimpleEchoServer = false,
PublisherRoutes = new() {
new("bluto://127.0.0.1:12345", ".*")
new("bts-cloud://127.0.0.1:12345", ".*")
},
SubscriberRoutes = new() {
new("bluto://127.0.0.1:12345", ".*")
new("bts-cloud://127.0.0.1:12345", ".*")
}
};
return rv;
@@ -193,30 +134,14 @@ public class PubSubConfig {
DoItAllSecondsBetweenLog ??= other.DoItAllSecondsBetweenLog;
HeartbeatInterval ??= other.HeartbeatInterval;
MissingHeartbeatsBeforeGc ??= other.MissingHeartbeatsBeforeGc;
UseManagedSockets ??= other.UseManagedSockets;
StateUpdatePeriod ??= other.StateUpdatePeriod;
StatisticsUpdatePeriod ??= other.StatisticsUpdatePeriod;
DisposedConnectionCheckPeriod ??= other.DisposedConnectionCheckPeriod;
MessageProcessingWarningThreshold ??= other.MessageProcessingWarningThreshold;
ServerPort ??= other.ServerPort;
MaxPacketSize ??= other.MaxPacketSize;
BlueCommMustDie ??= other.BlueCommMustDie;
CrushableTags ??= other.CrushableTags;
TagTypeToAssemblyMap ??= other.TagTypeToAssemblyMap;
ServerQueue ??= other.ServerQueue;
PortRange ??= other.PortRange;
BindToRandomPort ??= other.BindToRandomPort;
DebugTags ??= other.DebugTags;
LatencyReportThresholdMs ??= other.LatencyReportThresholdMs;
UseSubstations ??= other.UseSubstations;
IsSubstation ??= other.IsSubstation;
InternallyPublishSubInfo ??= other.InternallyPublishSubInfo;
UseManagedClientsInNativeCode ??= other.UseManagedClientsInNativeCode;
CompressedTags ??= other.CompressedTags;
TagsToReportCompressionRatio ??= other.TagsToReportCompressionRatio;
DisableMsmq ??= other.DisableMsmq;
UsePrepublishedServerState ??= other.UsePrepublishedServerState;
IpTranslation ??= other.IpTranslation;
GcActivationThreshold ??= other.GcActivationThreshold;
WaitPeriodAfterGc ??= other.WaitPeriodAfterGc;
SimpleEchoServer ??= other.SimpleEchoServer;
@@ -0,0 +1,98 @@
namespace bts {
using BCG;
using BCG.Utilities.General;
public class TableMetaDataStream : ViewStream, IWorker {
private class TableState {
public ViewStream table_stream;
public IBlueTableSchema schema = null;
public bool processed = false;
public TableState(ViewStream table_stream_) {
table_stream = table_stream_;
table_stream.onChange += _handle_table_update;
}
private void _handle_table_update(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
schema = dp.schema;
}
public void unsub() {
table_stream.onChange -= _handle_table_update;
}
}
private IBlueTableSchema _schema = BlueDataUtility.form_schema("TableMetaData",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("table_name", TypeCode.String),
Tuple.Create("col_idx", TypeCode.Int32),
Tuple.Create("col_type_code", TypeCode.Int32),
Tuple.Create("col_name", TypeCode.String));
private ViewStreamDatabase _vsd;
private ViewStream _base_stream;
private IDictionary<string, TableState> _table_map = new Dictionary<string, TableState>();
private BlueTable _state;
public TableMetaDataStream()
: this(ViewStreamDatabase.instance) { }
public TableMetaDataStream(ViewStreamDatabase vsd) {
_vsd = vsd;
_base_stream = new PubSubServerStateStream(_vsd);
_state = new BlueTable(_schema);
}
protected override void myOnFirstSubscriber() {
_base_stream.onChange += _handle_base_stream;
_vsd.addWorker(this);
}
protected override void myOnLastSubscriber() {
_base_stream.onChange -= _handle_base_stream;
_vsd.removeWorker(this);
}
public void doWork() {
bool changed = false;
foreach (TableState ts in _table_map.Values) {
if (!ts.processed) {
if (ts.schema != null) {
foreach (ColumnInfo ci in ts.schema.columnInfo) {
int id = _state.Count;
var table_name = ts.schema.tableName;
var col_idx = ci.colNo;
var col_name = ci.colName;
TypeCode type_code = BlueDataUtility.get_typecode(ci.colType);
BlueRow nr = new BlueRow(_schema, id, table_name, col_idx, Convert.ToInt32(type_code),
col_name);
changed = true;
_state.addRow(nr);
}
}
ts.unsub();
}
}
if (changed) {
var dp_out = BlueDataUtility.build_dp_inc(_state, myLastValue as DataPacket);
myFireEvent(dp_out);
}
}
private void _handle_base_stream(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
foreach (IBlueRow r in dp.newRows) {
string tag = r.get<string>("tag");
if (tag.StartsWith("DS_")) {
_handle_table(tag);
}
}
}
private void _handle_table(string tag_name) {
Log.assert(tag_name.StartsWith("DS_"));
string table_name = tag_name.Remove(0, 3);
TableState ts = null;
if (!_table_map.TryGetValue(table_name, out ts)) {
ts = new TableState(new TableStream(table_name, _vsd));
_table_map.Add(table_name, ts);
}
}
}
}