Added new benzinga streams
This commit is contained in:
1 parent
050e21e22b
commit
f06beff22f
19 files changed
+1108
-422
No files matched your search
@@ -1,23 +1,24 @@
|
||||
using bts;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// Provides a stream that with distinct rows with respect to the columns
|
||||
/// the user specifies in the argument
|
||||
/// </summary>
|
||||
[DataStreamUserRespresentation ("distinct")]
|
||||
public class DistinctRowStream : ViewStream
|
||||
{
|
||||
public DistinctRowStream (ViewStream stream, params string[] cols)
|
||||
{
|
||||
ArrayList groupByCols = new ArrayList ();
|
||||
foreach (string col in cols)
|
||||
foreach (string c in col.Split (' '))
|
||||
groupByCols.Add (c);
|
||||
mySetRealSource (new GroupByStream (stream,
|
||||
new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
|
||||
[DataStreamUserRespresentation("distinct")]
|
||||
public class DistinctRowStream : ViewStream {
|
||||
public DistinctRowStream(ViewStream stream, params string[] cols) {
|
||||
var g_by_eval = new DistinctRowEvaluator(cols);
|
||||
ViewStream r = new GroupByStream(stream, g_by_eval);
|
||||
var proj_exp = String.Empty;
|
||||
foreach (var c in cols)
|
||||
proj_exp += $"{c} ";
|
||||
proj_exp = proj_exp.Trim();
|
||||
r = new NewProjectStream(r, proj_exp);
|
||||
mySetRealSource(r);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,33 @@
|
||||
using BCG;
|
||||
|
||||
namespace bts {
|
||||
//[DataStreamUserRespresentation("saver")]
|
||||
// public class SaverStream : ViewStream {
|
||||
// private readonly ViewStream _vs;
|
||||
// private readonly string _file_name;
|
||||
// public SaverStream(ViewStream vs, string file_name) {
|
||||
// _vs = vs;
|
||||
// _file_name = file_name;
|
||||
// }
|
||||
|
||||
// protected override void myOnFirstSubscriber() {
|
||||
// _vs.onChange += _handle_dp;
|
||||
// }
|
||||
|
||||
// protected override void myOnLastSubscriber() {
|
||||
|
||||
// }
|
||||
|
||||
// private void _handle_dp(object dp_obj) {
|
||||
// DataPacket dp = dp_obj as DataPacket;
|
||||
// byte[] serialized_update = null;
|
||||
// var _serializer = new UniversalStreamIOSerializer();
|
||||
// serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
|
||||
// StreamSerializationUtils.write_to_file(serialized_update, _file_name);
|
||||
// myFireEvent(dp);
|
||||
// }
|
||||
// }
|
||||
|
||||
[DataStreamUserRespresentation("loader")]
|
||||
public class LoaderStream : ViewStream {
|
||||
|
||||
|
||||
@@ -31,7 +31,6 @@ namespace bts {
|
||||
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
|
||||
byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
|
||||
StreamSerializationUtils.write_to_file(str_bytes, _filename);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,26 +8,33 @@ using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
|
||||
namespace bts {
|
||||
public class PubSubTestingStream : ViewStream, IWorker {
|
||||
public class PayloadMessage {
|
||||
namespace bts
|
||||
{
|
||||
public class PubSubTestingStream : ViewStream, IWorker
|
||||
{
|
||||
public class PayloadMessage
|
||||
{
|
||||
public int msg_id = 0;
|
||||
public string guid;
|
||||
public Int64 travel_epoch_ms = 0;
|
||||
public byte[] payload = null;
|
||||
public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte) {
|
||||
public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
|
||||
{
|
||||
msg_id = msg_id_args;
|
||||
travel_epoch_ms = send_epoch_args;
|
||||
payload = new byte[payload_size];
|
||||
for (int i = 0; i < payload_size; i++) {
|
||||
for (int i = 0; i < payload_size; i++)
|
||||
{
|
||||
payload[i] = payload_byte;
|
||||
}
|
||||
}
|
||||
private PayloadMessage() { }
|
||||
public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
|
||||
this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte) { }
|
||||
this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
|
||||
{ }
|
||||
|
||||
public byte[] serialize() {
|
||||
public byte[] serialize()
|
||||
{
|
||||
var str = new MemoryStream();
|
||||
BinaryWriter bw = new BinaryWriter(str);
|
||||
bw.Write(msg_id);
|
||||
@@ -40,7 +47,8 @@ namespace bts {
|
||||
return bytes;
|
||||
}
|
||||
|
||||
public static PayloadMessage deserialize(byte[] ser_bytes) {
|
||||
public static PayloadMessage deserialize(byte[] ser_bytes)
|
||||
{
|
||||
var str = new MemoryStream(ser_bytes);
|
||||
BinaryReader br = new BinaryReader(str);
|
||||
int msg_id = br.ReadInt32();
|
||||
@@ -48,7 +56,8 @@ namespace bts {
|
||||
Int64 epoch = br.ReadInt64();
|
||||
int len = br.ReadInt32();
|
||||
var payload = br.ReadBytes(len);
|
||||
return new PayloadMessage() {
|
||||
return new PayloadMessage()
|
||||
{
|
||||
msg_id = msg_id,
|
||||
guid = guid,
|
||||
travel_epoch_ms = epoch,
|
||||
@@ -73,7 +82,8 @@ namespace bts {
|
||||
private readonly int _payload_sz;
|
||||
private const byte _payload_byte = (byte)'A';
|
||||
private readonly string _tag_name;
|
||||
public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
|
||||
public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
|
||||
{
|
||||
_vsd = vsd;
|
||||
_payload_sz = payload_size;
|
||||
var schema = BlueDataUtility.form_schema("PubSubTest",
|
||||
@@ -96,43 +106,53 @@ namespace bts {
|
||||
public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
|
||||
: this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
_vsd.addWorker(this);
|
||||
System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
|
||||
th.IsBackground = true;
|
||||
th.Start();
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
_vsd.removeWorker(this);
|
||||
_continue_sub_thread = false;
|
||||
}
|
||||
|
||||
void _log_updates(IEnumerable<PubSubServerMessage> updates) {
|
||||
void _log_updates(IEnumerable<PubSubServerMessage> updates)
|
||||
{
|
||||
var idx = 0;
|
||||
foreach (var msg in updates) {
|
||||
foreach (var msg in updates)
|
||||
{
|
||||
bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
|
||||
idx++;
|
||||
}
|
||||
}
|
||||
|
||||
private void _subscribe_thread_func() {
|
||||
private void _subscribe_thread_func()
|
||||
{
|
||||
// subscription is done in another thread
|
||||
SubscriberCore sub_core = new SubscriberCore();
|
||||
sub_core.subscribe(_tag_name);
|
||||
while (_continue_sub_thread) {
|
||||
while (_continue_sub_thread)
|
||||
{
|
||||
// TODO recv_id
|
||||
var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
|
||||
if (updates == null || updates.Count() == 0)
|
||||
continue;
|
||||
_log_updates(updates);
|
||||
_n_updates += updates.Count();
|
||||
if (updates.Count() > 0) {
|
||||
if (updates.Count() > 0)
|
||||
{
|
||||
var server_msg = updates.Last();
|
||||
if (server_msg.data != null && server_msg.data.Length > 0) {
|
||||
if (server_msg.data != null && server_msg.data.Length > 0)
|
||||
{
|
||||
_last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
|
||||
_last_recvd_msg.travel_epoch_ms = epoch_now();
|
||||
} else {
|
||||
}
|
||||
else
|
||||
{
|
||||
// why are we getting empty or null data ? (server is sending data regardless?)
|
||||
}
|
||||
_fire_dirty = true;
|
||||
@@ -140,29 +160,36 @@ namespace bts {
|
||||
}
|
||||
}
|
||||
|
||||
private void _publish_next_packet() {
|
||||
if (_last_sent_msg == null) {
|
||||
_last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte) {
|
||||
private void _publish_next_packet()
|
||||
{
|
||||
if (_last_sent_msg == null)
|
||||
{
|
||||
_last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
|
||||
{
|
||||
guid = Guid.NewGuid().ToString()
|
||||
};
|
||||
} else {
|
||||
}
|
||||
else
|
||||
{
|
||||
_last_sent_msg.msg_id++;
|
||||
_last_sent_msg.guid = Guid.NewGuid().ToString();
|
||||
_last_sent_msg.travel_epoch_ms = epoch_now();
|
||||
}
|
||||
var ser_bytes = _last_sent_msg.serialize();
|
||||
_n_pubs++;
|
||||
_pub_core.publish(String.Empty, _tag_name, ser_bytes);
|
||||
_pub_core.publish("Blob", _tag_name, ser_bytes);
|
||||
_fire_dirty = true;
|
||||
}
|
||||
|
||||
private void _fire_dp() {
|
||||
private void _fire_dp()
|
||||
{
|
||||
object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
|
||||
object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
|
||||
object echo_time = DBNull.Value;
|
||||
object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
|
||||
object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
|
||||
if (_last_sent_msg != null && _last_recvd_msg != null) {
|
||||
if (_last_sent_msg != null && _last_recvd_msg != null)
|
||||
{
|
||||
echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
|
||||
}
|
||||
var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
|
||||
@@ -174,21 +201,25 @@ namespace bts {
|
||||
myFireEvent(dp);
|
||||
}
|
||||
|
||||
public void doWork() {
|
||||
public void doWork()
|
||||
{
|
||||
if (DateTime.Now - _last_publish_time > _publish_period)
|
||||
if (_n_pubs <= _max_pubs) {
|
||||
if (_n_pubs <= _max_pubs)
|
||||
{
|
||||
_publish_next_packet();
|
||||
_last_publish_time = DateTime.Now;
|
||||
}
|
||||
// publish
|
||||
// TODO: send id
|
||||
if (_fire_dirty) {
|
||||
if (_fire_dirty)
|
||||
{
|
||||
_fire_dirty = false;
|
||||
_fire_dp();
|
||||
}
|
||||
}
|
||||
|
||||
public static long epoch_now() {
|
||||
public static long epoch_now()
|
||||
{
|
||||
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
using BCG;
|
||||
using BCG.Utilities.General;
|
||||
using System.Collections.Generic;
|
||||
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using bts.api;
|
||||
|
||||
namespace bts
|
||||
{
|
||||
[DataStreamUserRespresentation("pss_echo")]
|
||||
public class PubsubEchoTestStream : ViewStream, IWorker
|
||||
{
|
||||
public class PayloadMessage
|
||||
{
|
||||
public readonly string msg_id;
|
||||
public readonly Int64 timestamp = 0;
|
||||
public readonly byte[] payload = null;
|
||||
|
||||
public static byte[] create_data(int payload_size, byte payload_byte)
|
||||
{
|
||||
var rv = new byte[payload_size];
|
||||
for (int i = 0; i < payload_size; i++)
|
||||
{
|
||||
rv[i] = payload_byte;
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
// public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
|
||||
// {
|
||||
// guid = guid_;
|
||||
// timestamp = timestamp_;
|
||||
// payload = new byte[payload_size];
|
||||
// for (int i = 0; i < payload_size; i++)
|
||||
// {
|
||||
// payload[i] = payload_byte;
|
||||
// }
|
||||
// }
|
||||
public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
|
||||
{
|
||||
msg_id = guid_;
|
||||
timestamp = timestamp_;
|
||||
payload = payload_;
|
||||
}
|
||||
|
||||
private PayloadMessage() { }
|
||||
public byte[] serialize()
|
||||
{
|
||||
var str = new MemoryStream();
|
||||
BinaryWriter bw = new BinaryWriter(str);
|
||||
bw.Write(msg_id);
|
||||
bw.Write(timestamp);
|
||||
bw.Write(payload.Length);
|
||||
bw.Write(payload);
|
||||
str.Position = 0;
|
||||
var bytes = str.ToArray();
|
||||
return bytes;
|
||||
}
|
||||
|
||||
public static PayloadMessage deserialize(byte[] ser_bytes)
|
||||
{
|
||||
var str = new MemoryStream(ser_bytes);
|
||||
BinaryReader br = new BinaryReader(str);
|
||||
string guid = br.ReadString();
|
||||
Int64 timestamp_ = br.ReadInt64();
|
||||
int len = br.ReadInt32();
|
||||
var payload = br.ReadBytes(len);
|
||||
return new PayloadMessage(guid, timestamp_, payload);
|
||||
}
|
||||
}
|
||||
//public void
|
||||
private readonly int _msg_size;
|
||||
private readonly TimeSpan _publish_period;
|
||||
private DateTime _last_publish_time = DateTime.MinValue;
|
||||
private PublisherCore _pub_core;
|
||||
private ISubscriberCore _sub_core;
|
||||
private BlueTable _table;
|
||||
private int _n_pubs = 0;
|
||||
private readonly int _max_pubs;
|
||||
private int _n_updates = 0;
|
||||
private ViewStreamDatabase _vsd;
|
||||
private PayloadMessage _last_sent_msg, _last_recvd_msg;
|
||||
private const byte _payload_byte = (byte)'A';
|
||||
private readonly string _tag_name;
|
||||
private readonly int _payload_sz;
|
||||
private readonly IPubSubRouter _router;
|
||||
private enum IOState
|
||||
{
|
||||
SEND = 0,
|
||||
RECV = 1
|
||||
};
|
||||
IOState _state = IOState.SEND;
|
||||
|
||||
public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
|
||||
{
|
||||
var route = new PubSubRoute(server_url, ".*");
|
||||
_router = new PubSubRouter(route);
|
||||
_vsd = vsd;
|
||||
_payload_sz = payload_size;
|
||||
var schema = BlueDataUtility.form_schema("PubSubTest",
|
||||
Tuple.Create("id", TypeCode.Int32),
|
||||
Tuple.Create("sent_id", TypeCode.String),
|
||||
Tuple.Create("sent_time", TypeCode.Int64),
|
||||
Tuple.Create("recv_id", TypeCode.String),
|
||||
Tuple.Create("recv_time", TypeCode.Int64),
|
||||
Tuple.Create("match", TypeCode.Boolean),
|
||||
Tuple.Create("echo_time", TypeCode.Int32));
|
||||
_table = new BlueTable(schema);
|
||||
_publish_period = publish_period;
|
||||
_tag_name = tag_name;
|
||||
}
|
||||
|
||||
public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
|
||||
: this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
|
||||
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
_pub_core = new PublisherCore(_router);
|
||||
_sub_core = new SubscriberCore(_router);
|
||||
_sub_core.subscribe(_tag_name);
|
||||
_vsd.addWorker(this);
|
||||
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
_vsd.removeWorker(this);
|
||||
_sub_core.unsubscribe(_tag_name);
|
||||
}
|
||||
|
||||
private void _fire_dp_after_send()
|
||||
{
|
||||
// as soon as the data is published, its reflected in the stream state
|
||||
Log.assert(_last_sent_msg != null);
|
||||
BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
|
||||
DBNull.Value, DBNull.Value);
|
||||
_table.addRow(r);
|
||||
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
|
||||
myFireEvent(dp_out);
|
||||
}
|
||||
|
||||
private void _fire_dp_after_recv()
|
||||
{
|
||||
Log.assert(_last_recvd_msg != null);
|
||||
// update the last publish row
|
||||
int id = _table.Count - 1;
|
||||
IBlueRow last_row = _table[id];
|
||||
_table.deleteRow(last_row);
|
||||
var recv_ts = epoch_now();
|
||||
BlueRow nr = new BlueRow(_table.schema, id, _last_sent_msg.msg_id, _last_sent_msg.timestamp, _last_recvd_msg.msg_id, recv_ts,
|
||||
String.Compare(_last_recvd_msg.msg_id, _last_sent_msg.msg_id) == 0, recv_ts - _last_recvd_msg.timestamp);
|
||||
_table.addRow(nr);
|
||||
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
|
||||
myFireEvent(dp_out);
|
||||
}
|
||||
|
||||
private string _get_next_id()
|
||||
{
|
||||
return _table.Count.ToString();
|
||||
}
|
||||
|
||||
private void _publish_next_packet()
|
||||
{
|
||||
var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
|
||||
_last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
|
||||
var ser_bytes = _last_sent_msg.serialize();
|
||||
_n_pubs++;
|
||||
_pub_core.publish("Blob", _tag_name, ser_bytes);
|
||||
_last_publish_time = DateTime.Now;
|
||||
}
|
||||
|
||||
public void doWork()
|
||||
{
|
||||
Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
|
||||
{
|
||||
foreach (var u in updates)
|
||||
{
|
||||
var recv_msg = PayloadMessage.deserialize(u.data);
|
||||
if (recv_msg.msg_id == _last_sent_msg.msg_id)
|
||||
{
|
||||
return recv_msg;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
switch (_state)
|
||||
{
|
||||
case IOState.SEND:
|
||||
if (DateTime.Now - _last_publish_time >= _publish_period)
|
||||
{
|
||||
_publish_next_packet();
|
||||
_last_publish_time = DateTime.Now;
|
||||
_fire_dp_after_send();
|
||||
_state = IOState.RECV;
|
||||
return;
|
||||
}
|
||||
break;
|
||||
case IOState.RECV:
|
||||
var updates = _sub_core.get_updates(TimeSpan.Zero);
|
||||
updates = updates.Where(u => u.tag == _tag_name);
|
||||
var recv_msg = find_packet(updates);
|
||||
if (recv_msg != null)
|
||||
{
|
||||
_last_recvd_msg = recv_msg;
|
||||
_fire_dp_after_recv();
|
||||
_state = IOState.SEND;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
public static long epoch_now()
|
||||
{
|
||||
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
namespace bts {
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
[DataStreamUserRespresentation("benzingaNews")]
|
||||
public class BenzingaNewStream : ViewStream {
|
||||
private IBlueTableSchema _schema = null;
|
||||
private int _id = 0;
|
||||
// constructor takes in the raw mongo db stream, and the time to live (ttl), "7d" for example
|
||||
public BenzingaNewStream(ViewStream mongodb_raw_stream, string ttl) {
|
||||
// Hiding body (with project, columns can be hidden, or renamed)
|
||||
ViewStream output_stream = new ProjectStream(mongodb_raw_stream, "!body");
|
||||
// add column news_source = 'Benzinga', so we can append other news sources
|
||||
output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga");
|
||||
// now SuperMapStream is used to map one row into multiple, one per symbol,
|
||||
// (SuperMapStream takes 2 lambdas, one to get the enw schema and another to
|
||||
// return an array of rows, per one row)
|
||||
output_stream = new SuperMapStream(mongodb_raw_stream,
|
||||
|
||||
(IBlueTableSchema s) => {
|
||||
// this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange)
|
||||
if (_schema == null) {
|
||||
var out_schema = s;
|
||||
out_schema = BlueDataUtility.remove_columns(out_schema, "securities");
|
||||
out_schema = BlueDataUtility.add_column(s, "doc_id_internal", typeof(string));
|
||||
out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string));
|
||||
out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string));
|
||||
out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool));
|
||||
_schema = out_schema;
|
||||
}
|
||||
return _schema;
|
||||
},
|
||||
(IBlueRow r) => {
|
||||
var json_objs = BlueDataUtility.json_to_json_node_list(r.get<string>("securities"));
|
||||
List<IBlueRow> expanded_rows = new List<IBlueRow>();
|
||||
if (json_objs.Count == 0) {
|
||||
MutableBlueRow nr = new MutableBlueRow(_schema, r);
|
||||
nr.setValue("id", _id++);
|
||||
expanded_rows.Add(nr);
|
||||
}
|
||||
foreach (JsonNode jnode in json_objs) {
|
||||
if (jnode is JsonObject json_object) {
|
||||
MutableBlueRow nr = new MutableBlueRow(_schema, r);
|
||||
nr.setValue("id", _id++);
|
||||
nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
|
||||
nr.setValue("symbol", jnode["symbol"]);
|
||||
nr.setValue("exchange", jnode["exchange"]);
|
||||
nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
|
||||
expanded_rows.Add(nr);
|
||||
}
|
||||
}
|
||||
return expanded_rows;
|
||||
|
||||
});
|
||||
// now we creaate a TimeFilter stream to keep the table as a circular buffer of the last N days
|
||||
// passed A an argument in the constructor above
|
||||
// time_filter is a string like 7d 1h 30 m 45s
|
||||
output_stream = new TimeBasedFilterStream(output_stream, "created_at", ttl);
|
||||
mySetRealSource(output_stream);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,122 +7,127 @@ using MongoDB.Driver;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Threading.Tasks;
|
||||
using System.Threading;
|
||||
|
||||
namespace bts {
|
||||
[DataStreamUserRespresentation("mongodb_sql")]
|
||||
public class MongoDBSQLStream : ViewStream, IWorker {
|
||||
private ViewStreamDatabase _vsd;
|
||||
private readonly string _connection_string, _db_name, _collection_name;
|
||||
private MongoClient _client;
|
||||
// make this a sorted table
|
||||
private BlueTable _table;
|
||||
// [DataStreamUserRespresentation("mongodb_sql")]
|
||||
// public class MongoDBSQLStream : ViewStream, IWorker {
|
||||
// private ViewStreamDatabase _vsd;
|
||||
// private readonly string _connection_string, _db_name, _collection_name;
|
||||
// private MongoClient _client;
|
||||
// // make this a sorted table
|
||||
// private BlueTable _table;
|
||||
|
||||
|
||||
// Access the database
|
||||
private IMongoDatabase _database;
|
||||
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
||||
// mongodb://192.168.1.80:27017
|
||||
// // Access the database
|
||||
// private IMongoDatabase _database;
|
||||
// private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
||||
// // mongodb://192.168.1.80:27017
|
||||
|
||||
public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
|
||||
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
||||
// public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
|
||||
// : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
||||
|
||||
}
|
||||
public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
||||
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
||||
$"db_name: {db_name}");
|
||||
_connection_string = connection_string;
|
||||
_db_name = db_name;
|
||||
_collection_name = collection_name;
|
||||
_vsd = vsd;
|
||||
_client = new MongoClient(_connection_string);
|
||||
_database = _client.GetDatabase(_db_name);
|
||||
}
|
||||
// }
|
||||
// public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
||||
// bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
||||
// $"db_name: {db_name}");
|
||||
// _connection_string = connection_string;
|
||||
// _db_name = db_name;
|
||||
// _collection_name = collection_name;
|
||||
// _vsd = vsd;
|
||||
// _client = new MongoClient(_connection_string);
|
||||
// _database = _client.GetDatabase(_db_name);
|
||||
// }
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
|
||||
}
|
||||
// protected override void myOnFirstSubscriber() {
|
||||
// MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
|
||||
// }
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
MasterWorker.getInstance(false).removeWorker(this);
|
||||
}
|
||||
// protected override void myOnLastSubscriber() {
|
||||
// MasterWorker.getInstance(false).removeWorker(this);
|
||||
// }
|
||||
|
||||
public void doWork() {
|
||||
_read_mogodb();
|
||||
}
|
||||
// public void doWork() {
|
||||
// _read_mogodb();
|
||||
// }
|
||||
|
||||
private void _read_mogodb() {
|
||||
var timer = Stopwatch.StartNew();
|
||||
// Access the collection
|
||||
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
||||
// private void _read_mogodb() {
|
||||
// var timer = Stopwatch.StartNew();
|
||||
// // Access the collection
|
||||
// IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
||||
|
||||
// Fetch all documents from the collection
|
||||
var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
||||
_table = _table == null ? null : new BlueTable(_table.schema);
|
||||
// Iterate through the documents and print values
|
||||
foreach (var document in documents.ToEnumerable()) {
|
||||
bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
||||
if (_table == null) {
|
||||
_init_table(document);
|
||||
}
|
||||
_add_row(document);
|
||||
}
|
||||
timer.Stop();
|
||||
Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
|
||||
timer.Restart();
|
||||
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||
timer.Stop();
|
||||
Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
|
||||
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||
return;
|
||||
myFireEvent(dp);
|
||||
}
|
||||
// // Fetch all documents from the collection
|
||||
// var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
||||
// _table = _table == null ? null : new BlueTable(_table.schema);
|
||||
// // Iterate through the documents and print values
|
||||
// foreach (var document in documents.ToEnumerable()) {
|
||||
// bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
||||
// if (_table == null) {
|
||||
// _init_table(document);
|
||||
// }
|
||||
// _add_row(document);
|
||||
// }
|
||||
// timer.Stop();
|
||||
// Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
|
||||
// timer.Restart();
|
||||
// var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||
// timer.Stop();
|
||||
// Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
|
||||
// if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||
// return;
|
||||
// myFireEvent(dp);
|
||||
// }
|
||||
|
||||
private void _init_table(BsonDocument doc_row) {
|
||||
bts.Log.assert(_table == null);
|
||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||
cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
|
||||
foreach (var n in doc_row.Names) {
|
||||
cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
|
||||
}
|
||||
IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
|
||||
_table = new BlueTable(schema);
|
||||
}
|
||||
// private void _init_table(BsonDocument doc_row) {
|
||||
// bts.Log.assert(_table == null);
|
||||
// List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||
// cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
|
||||
// foreach (var n in doc_row.Names) {
|
||||
// cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
|
||||
// }
|
||||
// IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
|
||||
// _table = new BlueTable(schema);
|
||||
// }
|
||||
|
||||
private void _add_row(BsonDocument doc) {
|
||||
var key_val = doc["_id"];
|
||||
var key = key_val.ToString();
|
||||
int id;
|
||||
if (!_id_map.TryGetValue(key, out id)) {
|
||||
id = _id_map.Count;
|
||||
_id_map.Add(key, id);
|
||||
}
|
||||
MutableBlueRow nr = new MutableBlueRow(_table.schema);
|
||||
nr.setValue("id", id);
|
||||
foreach (var n in doc.Names) {
|
||||
var val = doc[n].ToString();
|
||||
nr.setValue(n, val);
|
||||
}
|
||||
_table.addRow(nr);
|
||||
}
|
||||
}
|
||||
// private void _add_row(BsonDocument doc) {
|
||||
// var key_val = doc["_id"];
|
||||
// var key = key_val.ToString();
|
||||
// int id;
|
||||
// if (!_id_map.TryGetValue(key, out id)) {
|
||||
// id = _id_map.Count;
|
||||
// _id_map.Add(key, id);
|
||||
// }
|
||||
// MutableBlueRow nr = new MutableBlueRow(_table.schema);
|
||||
// nr.setValue("id", id);
|
||||
// foreach (var n in doc.Names) {
|
||||
// var val = doc[n].ToString();
|
||||
// nr.setValue(n, val);
|
||||
// }
|
||||
// _table.addRow(nr);
|
||||
// }
|
||||
// }
|
||||
|
||||
[DataStreamUserRespresentation("mongodb")]
|
||||
public class MongoDBStream : ViewStream, IWorker {
|
||||
private ViewStreamDatabase _vsd;
|
||||
private readonly string _connection_string, _db_name, _collection_name;
|
||||
private readonly TimeSpan _poll_period;
|
||||
private MongoClient _client;
|
||||
private BlueTable _table;
|
||||
private SyncQueue<DataPacket> _table_queue = new SyncQueue<DataPacket>();
|
||||
private BlockingQueue _work_queue = new BlockingQueue();
|
||||
|
||||
// Access the database
|
||||
private IMongoDatabase _database;
|
||||
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
||||
// mongodb://192.168.1.80:27017
|
||||
|
||||
public MongoDBStream(string connection_string, string db_name, string collection_name)
|
||||
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
||||
public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period)
|
||||
: this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) {
|
||||
|
||||
}
|
||||
public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
||||
public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) {
|
||||
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
||||
$"db_name: {db_name}");
|
||||
_connection_string = connection_string;
|
||||
@@ -130,19 +135,30 @@ namespace bts {
|
||||
_collection_name = collection_name;
|
||||
_vsd = vsd;
|
||||
_client = new MongoClient(_connection_string);
|
||||
_poll_period = poll_period;
|
||||
_database = _client.GetDatabase(_db_name);
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
|
||||
_vsd.addWorker(this);
|
||||
//MasterWorker.getInstance(false).addWorker(this, _poll_period);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
MasterWorker.getInstance(false).removeWorker(this);
|
||||
_vsd.removeWorker(this);
|
||||
//MasterWorker.getInstance(false).removeWorker(this);
|
||||
}
|
||||
|
||||
private DateTime _last_poll_time = DateTime.MinValue;
|
||||
public void doWork() {
|
||||
_read_mogodb();
|
||||
var now = DateTime.Now;
|
||||
if (now - _last_poll_time > _poll_period) {
|
||||
ThreadPool.QueueUserWorkItem(_ => _read_mogodb());
|
||||
_last_poll_time = DateTime.Now;
|
||||
}
|
||||
var dp = _table_queue.dequeue(0);
|
||||
if (dp != null)
|
||||
myFireEvent(dp);
|
||||
}
|
||||
|
||||
private void _read_mogodb() {
|
||||
@@ -166,10 +182,11 @@ namespace bts {
|
||||
timer.Restart();
|
||||
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||
timer.Stop();
|
||||
Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms");;
|
||||
Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ;
|
||||
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||
return;
|
||||
myFireEvent(dp);
|
||||
_table_queue.enqueue(dp);
|
||||
//myFireEvent(dp);
|
||||
}
|
||||
|
||||
private void _init_table(BsonDocument doc_row) {
|
||||
@@ -208,46 +225,46 @@ namespace bts {
|
||||
}
|
||||
}
|
||||
|
||||
[DataStreamUserRespresentation("mongodb_flat")]
|
||||
public class MongoDBFlatStream : ViewStream {
|
||||
public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
|
||||
:
|
||||
this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
|
||||
// [DataStreamUserRespresentation("mongodb_flat")]
|
||||
// public class MongoDBFlatStream : ViewStream {
|
||||
// public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
|
||||
// :
|
||||
// this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
|
||||
|
||||
public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
|
||||
ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
||||
rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
|
||||
rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
|
||||
//rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
|
||||
mySetRealSource(rs);
|
||||
}
|
||||
}
|
||||
// public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
|
||||
// ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
||||
// rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
|
||||
// rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
|
||||
// //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
|
||||
// mySetRealSource(rs);
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
|
||||
[DataStreamUserRespresentation("newsreplay")]
|
||||
public class NewsReplayStream : ViewStream {
|
||||
public NewsReplayStream(string connection_string, string db_name, string collection_name,
|
||||
int buffer_sz, string fire_period) :
|
||||
this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
|
||||
// [DataStreamUserRespresentation("newsreplay")]
|
||||
// public class NewsReplayStream : ViewStream {
|
||||
// public NewsReplayStream(string connection_string, string db_name, string collection_name,
|
||||
// int buffer_sz, string fire_period) :
|
||||
// this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
|
||||
|
||||
public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
|
||||
int buffer_sz, TimeSpan fire_period) {
|
||||
ViewStream r;
|
||||
// public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
|
||||
// int buffer_sz, TimeSpan fire_period) {
|
||||
// ViewStream r;
|
||||
|
||||
r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
||||
r = new KeyValueToRelationalStream(r,
|
||||
"doc_id",
|
||||
"key",
|
||||
"value");
|
||||
r = new DateStandardizationStream(r,
|
||||
"created_at",
|
||||
"ISO8601");
|
||||
r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
|
||||
// r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
||||
// r = new KeyValueToRelationalStream(r,
|
||||
// "doc_id",
|
||||
// "key",
|
||||
// "value");
|
||||
// r = new DateStandardizationStream(r,
|
||||
// "created_at",
|
||||
// "ISO8601");
|
||||
// r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
|
||||
|
||||
mySetRealSource(r);
|
||||
}
|
||||
}
|
||||
// mySetRealSource(r);
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
}
|
||||
@@ -6,6 +6,7 @@ namespace bts {
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
public class ProjectData {
|
||||
// schema from the base stream
|
||||
@@ -147,12 +148,23 @@ namespace bts {
|
||||
mapped_to_orig_map.Add(mapped_col, orig_col);
|
||||
orig_to_mapped.Add(orig_col, mapped_col);
|
||||
} else {
|
||||
Log.Gen.error($"NewProjectStream unexpecetd token: {p_tok}");
|
||||
if (input_schema.columnNames.Contains(p_tok)) {
|
||||
string orig_col = p_tok;
|
||||
string mapped_col = orig_col;
|
||||
orig_to_mapped.Add(orig_col, mapped_col);
|
||||
}
|
||||
}
|
||||
}
|
||||
// make the new schema
|
||||
List<ColumnInfo> new_cols = new List<ColumnInfo>();
|
||||
foreach (ColumnInfo ci in input_schema.columnInfo) {
|
||||
if (ci.colName == "id") {
|
||||
// if there's a hidden col, other columns will show by default, but if cols are being
|
||||
// projected epxlicitely, than others wil be hidden
|
||||
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
||||
new_cols.Add(nc);
|
||||
continue;
|
||||
}
|
||||
if (removed_cols.Contains(ci.colName)) {
|
||||
continue;
|
||||
} else if (orig_to_mapped.ContainsKey(ci.colName)) {
|
||||
@@ -160,8 +172,12 @@ namespace bts {
|
||||
ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
|
||||
new_cols.Add(nc);
|
||||
} else {
|
||||
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
||||
new_cols.Add(nc);
|
||||
if (ci.colName == "id" || removed_cols.Count > 0) {
|
||||
// if there's a hidden col, other columns will show by default, but if cols are being
|
||||
// projected epxlicitely, than others wil be hidden
|
||||
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
||||
new_cols.Add(nc);
|
||||
}
|
||||
}
|
||||
}
|
||||
IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
|
||||
|
||||
Reference in new issue
Block a user