Added new benzinga streams

This commit is contained in:
Taha committed 2024-07-17 02:02:01 -04:00
1 parent 050e21e22b
commit f06beff22f
19 files changed
+1108 -422

No files matched your search

@@ -0,0 +1,85 @@
namespace bts {
using BCG;
using System;
using System.Collections.Generic;
using System.Net;
using System.Text.Json;
using System.Text.Json.Nodes;
[DataStreamUserRespresentation("benzingaNews")]
public class BenzingaNewsStream : ViewStream {
private IBlueTableSchema _schema = null;
private int _id = 0;
private static ViewStream _pre_process_raw_kvstream(ViewStream mongo_db_str) {
ViewStream r = new KeyValueToRelationalStream(mongo_db_str, "doc_id", "key", "value");
r = new DateStandardizationStream(r, "created_at", "ISO8601");
return r;
}
public BenzingaNewsStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl)
: this(new MongoDBStream(connection_string, db_name, collection_name, poll_period), ttl) {
}
// public BenzingaNewsStream()
// : this(new MongoDBStream(DataStreamPub connection_string, db_name, collection_name, poll_period), ttl) {
// }
// assumes it gets a raw mongodb_raw_kv_stream with key value pairs
public BenzingaNewsStream(ViewStream mongodb_raw_kv_stream, string ttl) {
ViewStream output_stream = _pre_process_raw_kvstream(mongodb_raw_kv_stream);
// Hiding body (with project, columns can be hidden, or renamed)
output_stream = new ProjectStream(output_stream, "!body");
// add column news_source = 'Benzinga', so we can append other news sources
output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga");
// now SuperMapStream is used to map one row into multiple, one per symbol,
// (SuperMapStream takes 2 lambdas, one to get the enw schema and another to
// return an array of rows, per one row)
output_stream = new SuperMapStream(output_stream,
(IBlueTableSchema s) => {
// this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange)
if (_schema == null) {
var out_schema = s;
out_schema = BlueDataUtility.remove_columns(out_schema, "securities");
out_schema = BlueDataUtility.remove_columns(out_schema, "body");
out_schema = BlueDataUtility.remove_columns(out_schema, "tags");
out_schema = BlueDataUtility.add_column(out_schema, "doc_id_internal", typeof(string));
out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string));
out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string));
out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool));
_schema = out_schema;
}
return _schema;
},
(IBlueRow r) => {
var json_objs = BlueDataUtility.json_to_json_node_list(r.get<string>("securities"));
List<IBlueRow> expanded_rows = new List<IBlueRow>();
if (json_objs.Count == 0) {
MutableBlueRow nr = new MutableBlueRow(_schema, r);
nr.setValue("id", _id++);
expanded_rows.Add(nr);
}
foreach (JsonNode jnode in json_objs) {
if (jnode is JsonObject json_object) {
MutableBlueRow nr = new MutableBlueRow(_schema, r);
nr.setValue("id", _id++);
nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
nr.setValue("symbol", jnode["symbol"]);
nr.setValue("exchange", jnode["exchange"]);
nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
expanded_rows.Add(nr);
}
}
return expanded_rows;
});
// now we creaate a TimeFilter stream to keep the table as a circular buffer of the last N days
// passed A an argument in the constructor above
// time_filter is a string like 7d 1h 30 m 45s
output_stream = new TimeBasedFilterStream(output_stream, "created_at", ttl);
mySetRealSource(output_stream);
}
}
}
@@ -0,0 +1,33 @@
namespace bts {
using BCG;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
//[DataStreamUserRespresentation("benzingaNewsSymbols")]
public class BenzingaNewsSymbolStateStream : ViewStream, INonIncGroupByEvaluator {
public BenzingaNewsSymbolStateStream(ViewStream news_str) {
ViewStream r = new NonIncGroupByStream(news_str, this);
mySetRealSource(r);
}
public BenzingaNewsSymbolStateStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl) :
this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) {
}
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
BlueTable rv = null;
foreach (var kv in grouped_rows) {
var symbol_rows = kv.Value;
IBlueRow state_row = BlueDataUtility.find_max(symbol_rows, "created_at");
rv = rv == null ? new BlueTable(state_row.schema) : rv;
rv.addRow(state_row);
}
return rv;
}
public string[] get_group_by_cols() {
return new string[] { "symbol" };
}
}
}
@@ -0,0 +1,14 @@
namespace bts {
using BCG;
[DataStreamUserRespresentation("benzingaNewsSymbols")]
public class BenzingaNewsSymbolsStream : ViewStream {
public BenzingaNewsSymbolsStream(ViewStream news_str) {
ViewStream rs = new DistinctRowStream(news_str, "symbol");
mySetRealSource(rs);
}
public BenzingaNewsSymbolsStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl) :
this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) {
}
}
}
@@ -0,0 +1,48 @@
namespace Bts.DataStreamPublisher;
using Bts.PubSub;
using CommandLine;
using System;
using System.Collections.Generic;
using System.IO;
using System.Text.Json;
using System.Text.Json.Serialization;
public class SharedStreamConfig {
[JsonPropertyName("mongodb_url")]
public string MONGO_DB_URL { get; set; }
[JsonPropertyName("mongodb_db_name")]
public string MONGO_DB_NAME { get; set; }
[JsonPropertyName("mongodb_db_collection")]
public string MONGO_DB_COLLECTION { get; set; }
[JsonPropertyName("benzinga_news_ttl")]
public string BENZINGA_NEWS_TTL { get; set; }
[JsonPropertyName("pubsub")]
public PubSubConfig? PubSub { get; set; }
/// <summary>
/// Merges options with those defined in `ConfigFile` (does not overwrite
/// non-null existing values).
/// </summary>
/// <exception cref="InvalidOperationException">
/// JsonSerializer.Deserialize returns null
/// </exception>
public void MergeWithConfigFile() {
// if (ConfigFile is null) {
// return;
// }
// var cfgString = File.ReadAllText(ConfigFile);
// var parsedCfg = JsonSerializer.Deserialize<DataStreamPublisherConfig>(cfgString) ??
// throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
// MergeWith(parsedCfg);
// ConfigFile = null;
}
}