Files
spark/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs
T
John Colvin ea0a1fbcd3 refactor: Load IDataStreamFactory(s) with dependency injection
So we can create ViewStreams that depend on other services, config, etc.
Previously, the PrimitiveManager relied purely on static construction of
types through reflection.
2025-11-19 13:53:18 -05:00

227 lines
10 KiB
C#

namespace bts.BlueOrb {
using bts;
using BCG;
using System.IO;
using System.Text;
using System.Runtime.InteropServices;
// OHLCV request breakdown
// symbol_or_contract (if symbol, get continous, otherwise get raw and filter it)
// bar_size (one min, one day)
// live/historic
// time range
// internal code oranization only
internal class OHLCV_1min_stream : ViewStream
{
public OHLCV_1min_stream(string symbol_or_contract, string range_beg, string range_end) :
this(symbol_or_contract, DateTime.Parse(range_beg), DateTime.Parse(range_end))
{ }
public OHLCV_1min_stream(string symbol_or_contract, DateTime range_beg, DateTime range_end)
{
Func<string, bool, SQLiteStream> get_sqlite_str = (symbol, is_continous_contracts) =>
{
string table_name = is_continous_contracts ? $"ohlcv_1min_{symbol}_continous" : $"{symbol}_ohlcv_1min";
string db_str = $"Data Source={SQLiteHelper.location}{SQLiteHelper.db_name};Version=3";
string range_beg_iso8601 = range_beg.ToString("yyyy-MM-ddTHH:mm:ss"),
range_end_iso8601 = range_end.ToString("yyyy-MM-ddTHH:mm:ss");
string sql_str = $"select * from {table_name} where DATE(ts_event) >= '{range_beg_iso8601}' AND DATE(ts_event) <= '{range_end_iso8601}'";
return new SQLiteStream(db_str, sql_str);
};
Tuple<ContractUtils.ContractCategory, string> tup = ContractUtils.get_symbol_category(symbol_or_contract);
var contract_cat = tup.Item1;
switch (contract_cat)
{
case ContractUtils.ContractCategory.Symbol:
{
ViewStream res = get_sqlite_str(tup.Item2, true);
mySetRealSource(res);
// this creates the continous stream for 1 min bars, but we store it in the DB, so we just read it from it
// ViewStream ohlcv_1min = get_sqlite_str(tup.Item2, false);
// // the following is whats written to the DB in the _contionous table
// ohlcv_1min = new DateTimeExpanstionStream(ohlcv_1min, "ts_event");
// ViewStream active_contr_str = new FuturesActiveContractStream(tup.Item2);
// ViewStream res = new NonIncJoinStream(ohlcv_1min, "ts_event_date symbol", "ts_event ts_event_date symbol open high low close volume",
// active_contr_str, "ts_event contract", "ts_event:date contract:continous_contract");
// res = new NullFilterStream(res, "continous_contract", true);
// res = new NewProjectStream(res, "ts_event date continous_contract open high low close volume !");
// mySetRealSource(res);
break;
}
default:
{
ViewStream sql_str = get_sqlite_str(tup.Item2, false);
ViewStream rs = new FilterStream(sql_str, $"symbol='{symbol_or_contract}'");
// filter here
mySetRealSource(rs);
break;
}
}
}
}
// this is to make bigger bars form smaller bars. Right now the assumption it only works
// with symbols vs contracts, and the 1min <= aggregation_period < 1day
//[DataStreamUserRespresentation("OHLCVAggregated")]
//public class OHLCVAggregatedStream : ViewStream {
// public OHLCVAggregatedStream(string symbol, string agregation_period, string range_beg, string range_end) :
// this(symbol, TimeBasedFilter.parse_duration(agregation_period), DateTime.Parse(range_beg), DateTime.Parse(range_end)) {
// }
// public OHLCVAggregatedStream(string symbol, TimeSpan aggregation_period, DateTime range_beg, DateTime range_end) {
// Tuple<ContractUtils.ContractCategory, string> tup = ContractUtils.get_symbol_category(symbol);
// Log.assert(tup.Item1 == ContractUtils.ContractCategory.Symbol);
// if (aggregation_period < TimeSpan.FromDays(1.0) &&
// aggregation_period > TimeSpan.FromMinutes(1.0)) {
// ViewStream r = new OHLCV_1min_stream(symbol, range_beg, range_end);
// r = new DateTimeExpanstionStream(r,
// "ts_event");
// double tot_secs = aggregation_period.TotalSeconds;
// r = new MathStream(r,
// $"bucket = Floor(ts_event_seconds_of_day / {tot_secs})");
// r = new NonIncGroupByStream(r,
// "bucket",
// // column:operation
// "open:first, high:max, low:min, close:last, volume:sum",
// "ts_event");
// r = new SortStream(r,
// "-ts_event");
// mySetRealSource(r);
// } else {
// // fix it later
// Log.assert(false);
// }
// }
//}
[DataStreamUserRespresentation("OHLCV")]
// This stream retreives the OHLCV table as is, and fires it as-is.
// The only logic here is as follows:
// 1. deciding bw sql table or pubsub
// 2. deciding bw live, historic or append(live, historic)
// 3. deciding size of bar
// (the logic for2 and 3 is in get_params)
public class OHLCVStream : ViewStream {
public enum Source {
SQL,
PubSub
}
public enum OHLCVType {
LIVE,
HISTORIC,
LIVE_AND_HISTORIC
};
public enum OHLCVBarSize {
ONE_DAY = 0,
ONE_HOUR,
ONE_MIN,
ONE_SEC
};
public class OHLCVParams {
public readonly Source source = Source.SQL;
public readonly OHLCVType type = OHLCVType.HISTORIC;
public readonly OHLCVBarSize bar_size;
public readonly Tuple<DateTime, DateTime> optional_dt_range;
public OHLCVParams(Source source_, OHLCVType type_, OHLCVBarSize bar_size_,
Tuple<DateTime, DateTime> optional_dt_range_) {
source = source_;
type = type_;
bar_size = bar_size_;
optional_dt_range = optional_dt_range_;
}
}
public static OHLCVParams get_params(DateTime dt1, DateTime dt2) {
Func<DateTime, bool> today = (dt) => dt.Date >= DateTime.Now.Date;
TimeSpan period = dt1 > dt2 ? dt1 - dt2 : dt2 - dt1;
bool has_today = today(dt1) || today(dt2);
bool has_history = !today(dt1) && !today(dt2);
Source source = Source.SQL;
OHLCVType type = (has_today && has_history) ? OHLCVType.LIVE_AND_HISTORIC :
((has_today && !has_history) ? OHLCVType.LIVE : OHLCVType.HISTORIC);
// TODO, for less than a day, its defaulted to 1 min
OHLCVBarSize bar_size = period >= TimeSpan.FromDays(1.0) ? OHLCVBarSize.ONE_DAY : OHLCVBarSize.ONE_MIN;
Tuple<DateTime, DateTime> optional_dt_range = new Tuple<DateTime, DateTime>(dt1, dt2);
return new OHLCVParams(source, type, bar_size, optional_dt_range);
}
public static Tuple<DateTime, DateTime> get_default_historic_range() {
return Tuple.Create(DateTime.Now.Date.Subtract(TimeSpan.FromDays(1)), DateTime.MinValue);
}
private static string get_table_from_barsize(OHLCVBarSize bar_size, string symbol) {
switch (bar_size) {
case OHLCVBarSize.ONE_DAY:
return $"{symbol}_OHLCV_1DAY";
case OHLCVBarSize.ONE_SEC:
case OHLCVBarSize.ONE_MIN:
case OHLCVBarSize.ONE_HOUR:
return $"{symbol}_OHLCV_1MIN";
}
Log.assert(false);
return String.Empty;
}
private Source _sourse = Source.SQL; // get from config
private readonly string _location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:\\BlueOrb_Data\\":Path.Combine(Environment.GetEnvironmentVariable("HOME") , "BlueOrb_Data");
private readonly string _db_name = "stream_data.db";
// the symbol is used ot retreive the correct table, the data flows as-is from the table
// with some name / type changes
public OHLCVStream(string symbol, OHLCVParams ohlcv_params) {
switch (ohlcv_params.type) {
case OHLCVType.LIVE:
case OHLCVType.LIVE_AND_HISTORIC:
// TODO
Log.assert(false);
break;
case OHLCVType.HISTORIC:
// TODO: assuming 1 day only for historic
string table_name = get_table_from_barsize(ohlcv_params.bar_size, symbol);// $"{symbol}_OHLCV_1DAY";
//string table_name = $"{symbol}_{ohlcv_params.bar_size}";
string db_name = _db_name;// $"Data Source={_location}{_db_name};Version=3";
string sql_str = $"select * from {table_name}";
// "Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;"
ViewStream rs = new SQLiteStream(db_name, sql_str);
rs = new SchemaGuesserStream(rs);
if (ohlcv_params.bar_size == OHLCVBarSize.ONE_DAY)
rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
rs = new NewProjectStream(rs, "symbol:contract");
mySetRealSource(rs);
break;
}
}
public OHLCVStream(string symbol)
:
this(symbol, new OHLCVParams(Source.SQL, OHLCVType.HISTORIC, OHLCVBarSize.ONE_DAY, get_default_historic_range())) { }
public OHLCVStream(string symbol, OHLCVBarSize bar_size)
:
this(symbol, new OHLCVParams(Source.SQL, OHLCVType.HISTORIC, bar_size, get_default_historic_range())) {
}
public OHLCVStream(string symbol, string bar_size)
:
this(symbol, Enum.Parse<OHLCVBarSize>(bar_size)) { }
// aggregates the bars
public OHLCVStream(string symbol, TimeSpan agregation_period) {
}
internal OHLCVStream(ViewStream source, OHLCVBarSize bar_size) {
ViewStream rs = source;
rs = new SchemaGuesserStream(rs);
if (bar_size == OHLCVBarSize.ONE_DAY) {
rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
}
rs = new NewProjectStream(rs, "symbol:contract");
mySetRealSource(rs);
}
}
}