improvement: Polish clis in DataStreamSuite
Specifically PubSubServer, DataStreamPublisher and DSQA * Use CommandLineParser (nuget pkg) * Clean up csproj files a bit while we're at it
This commit is contained in:
1 parent
63306173b1
commit
306c572b53
13 files changed
+501
-229
No files matched your search
@@ -1,130 +1,129 @@
|
||||
using System;
|
||||
using mbts.config;
|
||||
using bts.utils;
|
||||
using System.Threading.Tasks;
|
||||
using Bts.DataStreamExpressions;
|
||||
using System.Collections.Generic;
|
||||
|
||||
using BCG;
|
||||
using System.IO;
|
||||
using BCG.Utilities.General;
|
||||
using bts;
|
||||
using System.Linq;
|
||||
|
||||
|
||||
bts.Log.Gen.info($"Starting DSPublisher");
|
||||
Console.WriteLine("Starting DSPublisher...");
|
||||
bool run_echo_tcp_client_only = false;
|
||||
static void init(string[] args) {
|
||||
Func<string, string, Task> login = async (l, p) => {
|
||||
var login = l;
|
||||
var passwd = p;
|
||||
var service_ctx = bts.api.core.ServiceContext.create();
|
||||
await service_ctx.login(login, passwd);
|
||||
};
|
||||
BTSNativeMethods.provide_command_line(args, args.Length);
|
||||
Configuration.initialize();
|
||||
Task.Run(async () => await login("bts", String.Empty));
|
||||
}
|
||||
init(args);
|
||||
if (run_echo_tcp_client_only)
|
||||
start_echo_client();
|
||||
else
|
||||
_publish_streams();
|
||||
|
||||
static ViewStream build_stream(string stream_expression) {
|
||||
try {
|
||||
bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
|
||||
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
|
||||
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||
return rv;
|
||||
|
||||
} catch (Exception ex) {
|
||||
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
static void start_echo_client() {
|
||||
Console.WriteLine("Starting DSPublisher in TCP client echo mode, Hit key to continue");
|
||||
Console.ReadLine();
|
||||
SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345);
|
||||
client.start();
|
||||
while (true) {
|
||||
System.Threading.Thread.Sleep(1000);
|
||||
}
|
||||
// await SimpleTCPClient.start("127.0.0.1", 12345);
|
||||
// Task.Delay(1000000000).Wait();
|
||||
}
|
||||
|
||||
void _publish_streams() {
|
||||
PubTableBroker pb_broker = new PubTableBroker();
|
||||
CloudStreamManager csm = new CloudStreamManager();
|
||||
string csv_folder = @"C:\Temp\csvs_topublish";
|
||||
CSVPublisher csv_publisher = new CSVPublisher(csv_folder, pb_broker);
|
||||
// get it from config
|
||||
Dictionary<string, string> streams_to_publish = new Dictionary<string, string>() {
|
||||
//{"bbo","bbo()"},
|
||||
//{"instruments","instruments()"},
|
||||
//{"timesales","timesales(10000)"},
|
||||
//{"books","books()"},
|
||||
{"seconds_1_10","second(1,10)"},
|
||||
{"literal_420","literal(\"x\", \"420.0\")"},
|
||||
{"tradebar_30","tradebars(30)"},
|
||||
{"mongodb_testcollection", "mongodb(\"mongodb://192.168.1.80:27017\",\"local\",\"test_collection\")"}
|
||||
};
|
||||
// {"bbo_ts","timeSeries(bbo())"}
|
||||
|
||||
foreach (var kv in streams_to_publish) {
|
||||
var name = kv.Key;
|
||||
var exp = kv.Value;
|
||||
BCG.ViewStream vs = build_stream(exp);
|
||||
ViewPublisher vpub = new ViewPublisher(name, pb_broker, vs);
|
||||
vs.onChange += on_change;
|
||||
}
|
||||
while (true) {
|
||||
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
|
||||
ViewStreamDatabase.instance.doWork();
|
||||
csm.doWork();
|
||||
MasterWorker.getInstance(false).doWork();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
void on_change(object obj) {
|
||||
bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
|
||||
}
|
||||
|
||||
public class CSVPublisher : IWorker {
|
||||
private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
|
||||
// read from config
|
||||
private readonly string _csv_directory = @"C:\Temp\csvs_topublish\";
|
||||
|
||||
private PubTableBroker _pub_table_broker;
|
||||
public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
|
||||
_csv_directory = csv_directory;
|
||||
_pub_table_broker = pub_table_broker;
|
||||
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
|
||||
}
|
||||
public void doWork() {
|
||||
var file_names = Directory.GetFiles(_csv_directory);
|
||||
foreach (var fn in file_names) {
|
||||
if (!_pubs.ContainsKey(fn)) {
|
||||
_handle_new_file(fn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void _handle_new_file(string file_name) {
|
||||
if (file_name.EndsWith(".csv")) {
|
||||
ViewStream vs = new CSVReaderStream(file_name);
|
||||
vs = new SchemaGuesserStream(vs);
|
||||
var toks = file_name.Split('\\');
|
||||
string table_name = toks.Last();
|
||||
table_name = table_name.Remove(table_name.Length - ".csv".Length);
|
||||
ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs);
|
||||
_pubs.Add(file_name, vp);
|
||||
}
|
||||
}
|
||||
}
|
||||
using BCG;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using bts;
|
||||
using bts.api.core;
|
||||
using bts.utils;
|
||||
|
||||
using Bts.DataStreamExpressions;
|
||||
using Bts.DataStreamPublisher;
|
||||
|
||||
using CommandLine;
|
||||
|
||||
using mbts.config;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
await Parser.Default.ParseArguments<DataStreamPublisherConfig>(args)
|
||||
.MapResult(RunAsync, errs => Task.FromResult(1));
|
||||
|
||||
async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
|
||||
try {
|
||||
cfg.MergeWithConfigFile();
|
||||
Configuration.initialize(cfg.Sparkd);
|
||||
Log.Gen.info("Starting DataStreamPublisher");
|
||||
var service_ctx = ServiceContext.create(cfg.Sparkd);
|
||||
await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty);
|
||||
if (cfg.SimpleEchoClient ?? false) {
|
||||
start_echo_client();
|
||||
} else {
|
||||
_publish_streams(cfg.PrepublishTables ?? new(0));
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
// Write the error to console first, in case the logging system is compromised.
|
||||
var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
|
||||
Console.Error.WriteLine(error_msg);
|
||||
Console.Error.Flush();
|
||||
bts.Log.Gen.error(error_msg);
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static ViewStream build_stream(string stream_expression) {
|
||||
try {
|
||||
bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
|
||||
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
|
||||
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||
return rv;
|
||||
|
||||
} catch (Exception ex) {
|
||||
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
static void start_echo_client() {
|
||||
Console.WriteLine("Starting DSPublisher in TCP client echo mode, Hit key to continue");
|
||||
Console.ReadLine();
|
||||
SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345);
|
||||
client.start();
|
||||
while (true) {
|
||||
System.Threading.Thread.Sleep(1000);
|
||||
}
|
||||
// await SimpleTCPClient.start("127.0.0.1", 12345);
|
||||
// Task.Delay(1000000000).Wait();
|
||||
}
|
||||
|
||||
void _publish_streams(IEnumerable<KeyValuePair<string, string>> prepublish_streams) {
|
||||
PubTableBroker pb_broker = new PubTableBroker();
|
||||
CloudStreamManager csm = new CloudStreamManager();
|
||||
string csv_folder = @"C:\Temp\csvs_topublish";
|
||||
CSVPublisher csv_publisher = new CSVPublisher(csv_folder, pb_broker);
|
||||
foreach (var kv in prepublish_streams) {
|
||||
var name = kv.Key;
|
||||
var exp = kv.Value;
|
||||
BCG.ViewStream vs = build_stream(exp);
|
||||
ViewPublisher vpub = new ViewPublisher(name, pb_broker, vs);
|
||||
vs.onChange += on_change;
|
||||
}
|
||||
while (true) {
|
||||
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
|
||||
ViewStreamDatabase.instance.doWork();
|
||||
csm.doWork();
|
||||
MasterWorker.getInstance(false).doWork();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
void on_change(object obj) {
|
||||
bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
|
||||
}
|
||||
|
||||
public class CSVPublisher : IWorker {
|
||||
private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
|
||||
// read from config
|
||||
private readonly string _csv_directory = @"C:\Temp\csvs_topublish\";
|
||||
|
||||
private PubTableBroker _pub_table_broker;
|
||||
public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
|
||||
_csv_directory = csv_directory;
|
||||
_pub_table_broker = pub_table_broker;
|
||||
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
|
||||
}
|
||||
public void doWork() {
|
||||
var file_names = Directory.GetFiles(_csv_directory);
|
||||
foreach (var fn in file_names) {
|
||||
if (!_pubs.ContainsKey(fn)) {
|
||||
_handle_new_file(fn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void _handle_new_file(string file_name) {
|
||||
if (file_name.EndsWith(".csv")) {
|
||||
ViewStream vs = new CSVReaderStream(file_name);
|
||||
vs = new SchemaGuesserStream(vs);
|
||||
var toks = file_name.Split('\\');
|
||||
string table_name = toks.Last();
|
||||
table_name = table_name.Remove(table_name.Length - ".csv".Length);
|
||||
ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs);
|
||||
_pubs.Add(file_name, vp);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user