diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json index de5c21918..736a893e0 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json +++ b/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json @@ -9,6 +9,21 @@ "expression": "milliseconds(1000,10)", "is_on_websocket": true }, + { + "name": "benzinga_news", + "expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")", + "is_on_websocket": true + }, + { + "name": "benzinga_news_symbols", + "expression": "benzingaNewsSymbols(benzinga_news)", + "is_on_websocket": true + }, + { + "name": "benzinga_news_symbol_state", + "expression": "benzingaNewsSymbolState(benzinga_news)", + "is_on_websocket": true + }, { "name": "test_literal", "expression": "literal(\"x\", \"420.0\")", diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs index 11aab26c1..82f117b5c 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Program.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs @@ -23,8 +23,10 @@ using System.Threading.Tasks; await Parser.Default.ParseArguments(args) .MapResult(RunAsync, errs => Task.FromResult(1)); -async Task RunAsync(DataStreamPublisherConfig cfg) { - try { +async Task RunAsync(DataStreamPublisherConfig cfg) +{ + try + { cfg.MergeWithConfigFile(); cfg.MergeWith(DataStreamPublisherConfig.Default()); PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub)); @@ -33,7 +35,9 @@ async Task RunAsync(DataStreamPublisherConfig cfg) { var service_ctx = ServiceContext.create(cfg.Sparkd); await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty); _publish_streams(cfg.Publishers ?? new(0)); - } catch (Exception ex) { + } + catch (Exception ex) + { // Write the error to console first, in case the logging system is compromised. var error_msg = $"DataStreamPublisher threw exception:\n{ex}"; Console.Error.WriteLine(error_msg); @@ -44,29 +48,39 @@ async Task RunAsync(DataStreamPublisherConfig cfg) { return 0; } -static ViewStream? build_stream(string? stream_expression) { - if (stream_expression is null) { +static ViewStream? build_stream(string? stream_expression) +{ + // Console.WriteLine("Hit key to continue"); + // Console.ReadLine(); + if (stream_expression is null) + { return null; } - try { + try + { bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}"); var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); var rv = exp.getView(new Dictionary(0)); return rv; - } catch (Exception ex) { + } + catch (Exception ex) + { bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}"); return null; } } -void _publish_streams(IEnumerable publishers) { +void _publish_streams(IEnumerable publishers) +{ PubTableBroker pb_broker = new PubTableBroker(); RemoteQueryManager csm = new RemoteQueryManager(); var csvPublishers = new List(); - foreach (var publisher in publishers) { - if (Directory.Exists(publisher.CsvDir)) { + foreach (var publisher in publishers) + { + if (Directory.Exists(publisher.CsvDir)) + { csvPublishers.Add(new(publisher.CsvDir, pb_broker)); } } @@ -75,7 +89,8 @@ void _publish_streams(IEnumerable publishers) { .SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher)) .ToList(); - while (true) { + while (true) + { System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1)); ViewStreamDatabase.instance.doWork(); csm.doWork(); @@ -83,16 +98,19 @@ void _publish_streams(IEnumerable publishers) { ManagedSubscriber.instance.doWork(); } - ViewPublisher? createViewPublisher(PublishedStream ps) { + ViewPublisher? createViewPublisher(PublishedStream ps) + { Log.Gen.info( $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}"); - if (string.IsNullOrEmpty(ps.Name)) { + if (string.IsNullOrEmpty(ps.Name)) + { Log.Gen.warn( $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}"); return null; } var vs = build_stream(ps.Expression); - if (vs is null) { + if (vs is null) + { Log.Gen.warn( $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}"); return null; @@ -103,31 +121,39 @@ void _publish_streams(IEnumerable publishers) { } } -void on_change(object obj) { +void on_change(object obj) +{ bts.Log.Gen.info($"DSPublisher on_change dp: {obj}"); } -public class CSVPublisher : IWorker { +public class CSVPublisher : IWorker +{ private IDictionary _pubs = new Dictionary(); private readonly string _csv_directory; private PubTableBroker _pub_table_broker; - public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) { + public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) + { _csv_directory = csv_directory; _pub_table_broker = pub_table_broker; MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); } - public void doWork() { + public void doWork() + { var file_names = Directory.GetFiles(_csv_directory); - foreach (var fn in file_names) { - if (!_pubs.ContainsKey(fn)) { + foreach (var fn in file_names) + { + if (!_pubs.ContainsKey(fn)) + { _handle_new_file(fn); } } } - private void _handle_new_file(string file_name) { - if (file_name.EndsWith(".csv")) { + private void _handle_new_file(string file_name) + { + if (file_name.EndsWith(".csv")) + { ViewStream vs = new CSVReaderStream(file_name); vs = new SchemaGuesserStream(vs); var toks = file_name.Split('\\'); diff --git a/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsStream.cs b/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsStream.cs index d253fff9b..2b0868bef 100644 --- a/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsStream.cs @@ -63,11 +63,13 @@ namespace bts { } foreach (JsonNode jnode in json_objs) { if (jnode is JsonObject json_object) { + string symbol = jnode["symbol"] == null ? String.Empty : jnode["symbol"].ToString(); + string exchange = jnode["exchange"] == null ? String.Empty : jnode["exchange"].ToString(); MutableBlueRow nr = new MutableBlueRow(_schema, r); nr.setValue("id", _id++); nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}"); - nr.setValue("symbol", jnode["symbol"]); - nr.setValue("exchange", jnode["exchange"]); + nr.setValue("symbol", symbol); + nr.setValue("exchange", exchange); nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString())); expanded_rows.Add(nr); }