RowDiff added

This commit is contained in:
Taha Afzal committed 2024-02-08 14:11:10 -05:00
commit 8c1342230b
16 files changed
+628 -307

No files matched your search

@@ -6,7 +6,10 @@ using Bts.DataStreamExpressions;
using System.Collections.Generic;
using BCG;
using System.Threading;
using System.IO;
using BCG.Utilities.General;
using bts;
using System.Linq;
bts.Log.Gen.info($"Starting DSPublisher");
@@ -57,6 +60,8 @@ static void start_echo_client() {
void _publish_streams() {
PubTableBroker pb_broker = new PubTableBroker();
CloudStreamManager csm = new CloudStreamManager();
string csv_folder = @"C:\Temp\csvs_topublish";
CSVPublisher csv_publisher = new CSVPublisher(csv_folder, pb_broker);
// get it from config
Dictionary<string, string> streams_to_publish = new Dictionary<string, string>() {
//{"bbo","bbo()"},
@@ -88,4 +93,37 @@ void _publish_streams() {
void on_change(object obj) {
bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
}
public class CSVPublisher : IWorker {
private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
// read from config
private readonly string _csv_directory = @"C:\Temp\csvs_topublish\";
private PubTableBroker _pub_table_broker;
public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
_csv_directory = csv_directory;
_pub_table_broker = pub_table_broker;
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
}
public void doWork() {
var file_names = Directory.GetFiles(_csv_directory);
foreach (var fn in file_names) {
if (!_pubs.ContainsKey(fn)) {
_handle_new_file(fn);
}
}
}
private void _handle_new_file(string file_name) {
if (file_name.EndsWith(".csv")) {
ViewStream vs = new CSVReaderStream(file_name);
vs = new SchemaGuesserStream(vs);
var toks = file_name.Split('\\');
string table_name = toks.Last();
table_name = table_name.Remove(table_name.Length - ".csv".Length);
ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs);
_pubs.Add(file_name, vp);
}
}
}