improvement: Polish clis in DataStreamSuite

Specifically PubSubServer, DataStreamPublisher and DSQA

* Use CommandLineParser (nuget pkg)
* Clean up csproj files a bit while we're at it
This commit is contained in:
John Colvin committed 2024-06-03 13:46:51 +00:00
1 parent 63306173b1
commit 306c572b53
13 files changed
+501 -229

No files matched your search

+129 -130
View File
@@ -1,130 +1,129 @@
using System;
using mbts.config;
using bts.utils;
using System.Threading.Tasks;
using Bts.DataStreamExpressions;
using System.Collections.Generic;
using BCG;
using System.IO;
using BCG.Utilities.General;
using bts;
using System.Linq;
bts.Log.Gen.info($"Starting DSPublisher");
Console.WriteLine("Starting DSPublisher...");
bool run_echo_tcp_client_only = false;
static void init(string[] args) {
Func<string, string, Task> login = async (l, p) => {
var login = l;
var passwd = p;
var service_ctx = bts.api.core.ServiceContext.create();
await service_ctx.login(login, passwd);
};
BTSNativeMethods.provide_command_line(args, args.Length);
Configuration.initialize();
Task.Run(async () => await login("bts", String.Empty));
}
init(args);
if (run_echo_tcp_client_only)
start_echo_client();
else
_publish_streams();
static ViewStream build_stream(string stream_expression) {
try {
bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv;
} catch (Exception ex) {
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
return null;
}
}
static void start_echo_client() {
Console.WriteLine("Starting DSPublisher in TCP client echo mode, Hit key to continue");
Console.ReadLine();
SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345);
client.start();
while (true) {
System.Threading.Thread.Sleep(1000);
}
// await SimpleTCPClient.start("127.0.0.1", 12345);
// Task.Delay(1000000000).Wait();
}
void _publish_streams() {
PubTableBroker pb_broker = new PubTableBroker();
CloudStreamManager csm = new CloudStreamManager();
string csv_folder = @"C:\Temp\csvs_topublish";
CSVPublisher csv_publisher = new CSVPublisher(csv_folder, pb_broker);
// get it from config
Dictionary<string, string> streams_to_publish = new Dictionary<string, string>() {
//{"bbo","bbo()"},
//{"instruments","instruments()"},
//{"timesales","timesales(10000)"},
//{"books","books()"},
{"seconds_1_10","second(1,10)"},
{"literal_420","literal(\"x\", \"420.0\")"},
{"tradebar_30","tradebars(30)"},
{"mongodb_testcollection", "mongodb(\"mongodb://192.168.1.80:27017\",\"local\",\"test_collection\")"}
};
// {"bbo_ts","timeSeries(bbo())"}
foreach (var kv in streams_to_publish) {
var name = kv.Key;
var exp = kv.Value;
BCG.ViewStream vs = build_stream(exp);
ViewPublisher vpub = new ViewPublisher(name, pb_broker, vs);
vs.onChange += on_change;
}
while (true) {
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
ViewStreamDatabase.instance.doWork();
csm.doWork();
MasterWorker.getInstance(false).doWork();
}
}
void on_change(object obj) {
bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
}
public class CSVPublisher : IWorker {
private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
// read from config
private readonly string _csv_directory = @"C:\Temp\csvs_topublish\";
private PubTableBroker _pub_table_broker;
public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
_csv_directory = csv_directory;
_pub_table_broker = pub_table_broker;
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
}
public void doWork() {
var file_names = Directory.GetFiles(_csv_directory);
foreach (var fn in file_names) {
if (!_pubs.ContainsKey(fn)) {
_handle_new_file(fn);
}
}
}
private void _handle_new_file(string file_name) {
if (file_name.EndsWith(".csv")) {
ViewStream vs = new CSVReaderStream(file_name);
vs = new SchemaGuesserStream(vs);
var toks = file_name.Split('\\');
string table_name = toks.Last();
table_name = table_name.Remove(table_name.Length - ".csv".Length);
ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs);
_pubs.Add(file_name, vp);
}
}
}
using BCG;
using BCG.Utilities.General;
using bts;
using bts.api.core;
using bts.utils;
using Bts.DataStreamExpressions;
using Bts.DataStreamPublisher;
using CommandLine;
using mbts.config;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
await Parser.Default.ParseArguments<DataStreamPublisherConfig>(args)
.MapResult(RunAsync, errs => Task.FromResult(1));
async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
try {
cfg.MergeWithConfigFile();
Configuration.initialize(cfg.Sparkd);
Log.Gen.info("Starting DataStreamPublisher");
var service_ctx = ServiceContext.create(cfg.Sparkd);
await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty);
if (cfg.SimpleEchoClient ?? false) {
start_echo_client();
} else {
_publish_streams(cfg.PrepublishTables ?? new(0));
}
} catch (Exception ex) {
// Write the error to console first, in case the logging system is compromised.
var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
Console.Error.WriteLine(error_msg);
Console.Error.Flush();
bts.Log.Gen.error(error_msg);
return 1;
}
return 0;
}
static ViewStream build_stream(string stream_expression) {
try {
bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv;
} catch (Exception ex) {
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
return null;
}
}
static void start_echo_client() {
Console.WriteLine("Starting DSPublisher in TCP client echo mode, Hit key to continue");
Console.ReadLine();
SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345);
client.start();
while (true) {
System.Threading.Thread.Sleep(1000);
}
// await SimpleTCPClient.start("127.0.0.1", 12345);
// Task.Delay(1000000000).Wait();
}
void _publish_streams(IEnumerable<KeyValuePair<string, string>> prepublish_streams) {
PubTableBroker pb_broker = new PubTableBroker();
CloudStreamManager csm = new CloudStreamManager();
string csv_folder = @"C:\Temp\csvs_topublish";
CSVPublisher csv_publisher = new CSVPublisher(csv_folder, pb_broker);
foreach (var kv in prepublish_streams) {
var name = kv.Key;
var exp = kv.Value;
BCG.ViewStream vs = build_stream(exp);
ViewPublisher vpub = new ViewPublisher(name, pb_broker, vs);
vs.onChange += on_change;
}
while (true) {
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
ViewStreamDatabase.instance.doWork();
csm.doWork();
MasterWorker.getInstance(false).doWork();
}
}
void on_change(object obj) {
bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
}
public class CSVPublisher : IWorker {
private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
// read from config
private readonly string _csv_directory = @"C:\Temp\csvs_topublish\";
private PubTableBroker _pub_table_broker;
public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
_csv_directory = csv_directory;
_pub_table_broker = pub_table_broker;
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
}
public void doWork() {
var file_names = Directory.GetFiles(_csv_directory);
foreach (var fn in file_names) {
if (!_pubs.ContainsKey(fn)) {
_handle_new_file(fn);
}
}
}
private void _handle_new_file(string file_name) {
if (file_name.EndsWith(".csv")) {
ViewStream vs = new CSVReaderStream(file_name);
vs = new SchemaGuesserStream(vs);
var toks = file_name.Split('\\');
string table_name = toks.Last();
table_name = table_name.Remove(table_name.Length - ".csv".Length);
ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs);
_pubs.Add(file_name, vp);
}
}
}