Added new benzinga streams
This commit is contained in:
1 parent
050e21e22b
commit
f06beff22f
19 files changed
+1108
-422
No files matched your search
@@ -0,0 +1,85 @@
|
||||
namespace bts {
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
[DataStreamUserRespresentation("benzingaNews")]
|
||||
public class BenzingaNewsStream : ViewStream {
|
||||
private IBlueTableSchema _schema = null;
|
||||
private int _id = 0;
|
||||
|
||||
|
||||
private static ViewStream _pre_process_raw_kvstream(ViewStream mongo_db_str) {
|
||||
ViewStream r = new KeyValueToRelationalStream(mongo_db_str, "doc_id", "key", "value");
|
||||
r = new DateStandardizationStream(r, "created_at", "ISO8601");
|
||||
return r;
|
||||
}
|
||||
|
||||
public BenzingaNewsStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl)
|
||||
: this(new MongoDBStream(connection_string, db_name, collection_name, poll_period), ttl) {
|
||||
}
|
||||
|
||||
// public BenzingaNewsStream()
|
||||
// : this(new MongoDBStream(DataStreamPub connection_string, db_name, collection_name, poll_period), ttl) {
|
||||
// }
|
||||
|
||||
// assumes it gets a raw mongodb_raw_kv_stream with key value pairs
|
||||
public BenzingaNewsStream(ViewStream mongodb_raw_kv_stream, string ttl) {
|
||||
ViewStream output_stream = _pre_process_raw_kvstream(mongodb_raw_kv_stream);
|
||||
// Hiding body (with project, columns can be hidden, or renamed)
|
||||
output_stream = new ProjectStream(output_stream, "!body");
|
||||
// add column news_source = 'Benzinga', so we can append other news sources
|
||||
output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga");
|
||||
// now SuperMapStream is used to map one row into multiple, one per symbol,
|
||||
// (SuperMapStream takes 2 lambdas, one to get the enw schema and another to
|
||||
// return an array of rows, per one row)
|
||||
output_stream = new SuperMapStream(output_stream,
|
||||
|
||||
(IBlueTableSchema s) => {
|
||||
// this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange)
|
||||
if (_schema == null) {
|
||||
var out_schema = s;
|
||||
out_schema = BlueDataUtility.remove_columns(out_schema, "securities");
|
||||
out_schema = BlueDataUtility.remove_columns(out_schema, "body");
|
||||
out_schema = BlueDataUtility.remove_columns(out_schema, "tags");
|
||||
out_schema = BlueDataUtility.add_column(out_schema, "doc_id_internal", typeof(string));
|
||||
out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string));
|
||||
out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string));
|
||||
out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool));
|
||||
_schema = out_schema;
|
||||
}
|
||||
return _schema;
|
||||
},
|
||||
(IBlueRow r) => {
|
||||
var json_objs = BlueDataUtility.json_to_json_node_list(r.get<string>("securities"));
|
||||
List<IBlueRow> expanded_rows = new List<IBlueRow>();
|
||||
if (json_objs.Count == 0) {
|
||||
MutableBlueRow nr = new MutableBlueRow(_schema, r);
|
||||
nr.setValue("id", _id++);
|
||||
expanded_rows.Add(nr);
|
||||
}
|
||||
foreach (JsonNode jnode in json_objs) {
|
||||
if (jnode is JsonObject json_object) {
|
||||
MutableBlueRow nr = new MutableBlueRow(_schema, r);
|
||||
nr.setValue("id", _id++);
|
||||
nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
|
||||
nr.setValue("symbol", jnode["symbol"]);
|
||||
nr.setValue("exchange", jnode["exchange"]);
|
||||
nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
|
||||
expanded_rows.Add(nr);
|
||||
}
|
||||
}
|
||||
return expanded_rows;
|
||||
|
||||
});
|
||||
// now we creaate a TimeFilter stream to keep the table as a circular buffer of the last N days
|
||||
// passed A an argument in the constructor above
|
||||
// time_filter is a string like 7d 1h 30 m 45s
|
||||
output_stream = new TimeBasedFilterStream(output_stream, "created_at", ttl);
|
||||
mySetRealSource(output_stream);
|
||||
}
|
||||
}
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
namespace bts {
|
||||
using BCG;
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
|
||||
//[DataStreamUserRespresentation("benzingaNewsSymbols")]
|
||||
public class BenzingaNewsSymbolStateStream : ViewStream, INonIncGroupByEvaluator {
|
||||
public BenzingaNewsSymbolStateStream(ViewStream news_str) {
|
||||
ViewStream r = new NonIncGroupByStream(news_str, this);
|
||||
mySetRealSource(r);
|
||||
}
|
||||
public BenzingaNewsSymbolStateStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl) :
|
||||
this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) {
|
||||
}
|
||||
|
||||
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
|
||||
BlueTable rv = null;
|
||||
foreach (var kv in grouped_rows) {
|
||||
var symbol_rows = kv.Value;
|
||||
IBlueRow state_row = BlueDataUtility.find_max(symbol_rows, "created_at");
|
||||
rv = rv == null ? new BlueTable(state_row.schema) : rv;
|
||||
rv.addRow(state_row);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
public string[] get_group_by_cols() {
|
||||
return new string[] { "symbol" };
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
namespace bts {
|
||||
using BCG;
|
||||
[DataStreamUserRespresentation("benzingaNewsSymbols")]
|
||||
|
||||
public class BenzingaNewsSymbolsStream : ViewStream {
|
||||
public BenzingaNewsSymbolsStream(ViewStream news_str) {
|
||||
ViewStream rs = new DistinctRowStream(news_str, "symbol");
|
||||
mySetRealSource(rs);
|
||||
}
|
||||
public BenzingaNewsSymbolsStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl) :
|
||||
this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) {
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
namespace Bts.DataStreamPublisher;
|
||||
|
||||
using Bts.PubSub;
|
||||
|
||||
using CommandLine;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
public class SharedStreamConfig {
|
||||
|
||||
[JsonPropertyName("mongodb_url")]
|
||||
public string MONGO_DB_URL { get; set; }
|
||||
|
||||
[JsonPropertyName("mongodb_db_name")]
|
||||
public string MONGO_DB_NAME { get; set; }
|
||||
|
||||
[JsonPropertyName("mongodb_db_collection")]
|
||||
public string MONGO_DB_COLLECTION { get; set; }
|
||||
|
||||
[JsonPropertyName("benzinga_news_ttl")]
|
||||
public string BENZINGA_NEWS_TTL { get; set; }
|
||||
|
||||
[JsonPropertyName("pubsub")]
|
||||
public PubSubConfig? PubSub { get; set; }
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Merges options with those defined in `ConfigFile` (does not overwrite
|
||||
/// non-null existing values).
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidOperationException">
|
||||
/// JsonSerializer.Deserialize returns null
|
||||
/// </exception>
|
||||
public void MergeWithConfigFile() {
|
||||
// if (ConfigFile is null) {
|
||||
// return;
|
||||
// }
|
||||
// var cfgString = File.ReadAllText(ConfigFile);
|
||||
// var parsedCfg = JsonSerializer.Deserialize<DataStreamPublisherConfig>(cfgString) ??
|
||||
// throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
|
||||
// MergeWith(parsedCfg);
|
||||
// ConfigFile = null;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user