diff --git a/src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs b/src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs index effb56f1d..d3714466c 100644 --- a/src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs +++ b/src/DataStreamSuite/DataStream/Offloading/CloudStreamManager.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; - +using System.IO; +using System.Text; +using System.Runtime.InteropServices; using Bts.DataStreamExpressions; using BCG.Utilities.General; using bts; @@ -72,7 +74,10 @@ namespace BCG { ViewStream stream = build_stream(stream_expression); _expose_to_ws(name, stream); if (persist) { - stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs + string tempDir = Path.GetTempPath(); + Directory.CreateDirectory(tempDir); // make sure it exists + + stream = new PersistorStream(stream, name, tempDir, 10_000_000); // 10 MB } ViewPublisher view_pub = new ViewPublisher(name, _broker, stream); StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub); diff --git a/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs index b9610e3fc..3b475b051 100644 --- a/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs +++ b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs @@ -3,6 +3,8 @@ using System.Data; using System.Data.SQLite; using System.Diagnostics; using System.Collections; +using System.IO; +using System.Text; using BCG; @@ -10,12 +12,14 @@ using System.Linq; using System.Data.Entity; using System.Collections.Generic; using System.Data.SQLite; +using System.Runtime.InteropServices; + namespace bts.BlueOrb{ public static class SQLiteHelper { - public static string location = "C:\\BlueOrb_Data\\"; + public static string location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:\\BlueOrb_Data\\" : Path.Combine(Environment.GetEnvironmentVariable("HOME"), "BlueOrb_Data"); public static string db_name = "stream_data.db"; public static DataTable execute_select_query(string connection_string, string query) { diff --git a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs index 12324e741..aa132b1b2 100644 --- a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs @@ -117,6 +117,7 @@ namespace BCG { //_stopwatch.Stop(); //bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs"); _stopwatch.Start(); + bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber opening address{_file_address}"); IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address); _stopwatch.Stop(); bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs"); diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveBBOStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveBBOStream.cs index 7adf95c9c..4f3e6de34 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveBBOStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveBBOStream.cs @@ -62,7 +62,7 @@ } protected override void myOnFirstSubscriber() { - var file_path = $"{SQLiteHelper.location}\\mbp1_{_fcc.symbol}_small.csv"; + var file_path = Path.Combine(SQLiteHelper.location, $"mbp1_{_fcc.symbol}_small.csv".ToLower()); var csv_rd = new CSVReaderStream(file_path, false); _rows.rows = csv_rd.rows.to_list().OrderBy(r => r.get("ts_event")).ToList(); _rows.pointer = 0; diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveTradeStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveTradeStream.cs index d40755b42..2cda5019d 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveTradeStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveTradeStream.cs @@ -61,7 +61,7 @@ } protected override void myOnFirstSubscriber() { - var file_path = $"{SQLiteHelper.location}\\trades_{_fcc.symbol}.csv"; + var file_path = Path.Combine(SQLiteHelper.location, $"trades_{_fcc.symbol}.csv"); var csv_rd = new CSVReaderStream(file_path, false); _rows.rows = csv_rd.rows.to_list().OrderBy(r => r.get("ts_event")).ToList(); _rows.pointer = 0; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs index 2ec4abc0d..ba3f58b38 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs @@ -1,13 +1,17 @@ namespace bts.BlueOrb { using BCG; + using System.IO; + using System.Text; + using System.Runtime.InteropServices; [DataStreamUserRespresentation("Events")] public class EventsStream : ViewStream { public EventsStream(string event_type) : this(ViewStreamDatabase.instance, event_type) { } public EventsStream(ViewStreamDatabase vsd, string event_type) { + string path = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:\\BlueOrb_Data\\":Path.Combine(Environment.GetEnvironmentVariable("HOME") , "BlueOrb_Data"); ViewStream rs = - new SQLiteStream("Data Source=C:\\Users\\m_tah\\development\\data\\stream_data.db;Version=3;", + new SQLiteStream("Data Source="+ Path.Combine(path,"stream_data.db")+";Version=3;", "select * from events"); if (!String.IsNullOrEmpty(event_type)) { rs = new FilterStream(rs, r => String.Compare(event_type, r.get("event_type"), true) == 0); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs index 26069e458..ae5523fb3 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs @@ -1,6 +1,9 @@ namespace bts.BlueOrb { using bts; using BCG; + using System.IO; + using System.Text; + using System.Runtime.InteropServices; // OHLCV request breakdown // symbol_or_contract (if symbol, get continous, otherwise get raw and filter it) @@ -9,13 +12,17 @@ namespace bts.BlueOrb { // time range // internal code oranization only - internal class OHLCV_1min_stream : ViewStream { + internal class OHLCV_1min_stream : ViewStream + { public OHLCV_1min_stream(string symbol_or_contract, string range_beg, string range_end) : - this(symbol_or_contract, DateTime.Parse(range_beg), DateTime.Parse(range_end)) { } + this(symbol_or_contract, DateTime.Parse(range_beg), DateTime.Parse(range_end)) + { } - public OHLCV_1min_stream(string symbol_or_contract, DateTime range_beg, DateTime range_end) { - Func get_sqlite_str = (symbol, is_continous_contracts) => { + public OHLCV_1min_stream(string symbol_or_contract, DateTime range_beg, DateTime range_end) + { + Func get_sqlite_str = (symbol, is_continous_contracts) => + { string table_name = is_continous_contracts ? $"ohlcv_1min_{symbol}_continous" : $"{symbol}_ohlcv_1min"; string db_str = $"Data Source={SQLiteHelper.location}{SQLiteHelper.db_name};Version=3"; string range_beg_iso8601 = range_beg.ToString("yyyy-MM-ddTHH:mm:ss"), @@ -26,8 +33,10 @@ namespace bts.BlueOrb { Tuple tup = ContractUtils.get_symbol_category(symbol_or_contract); var contract_cat = tup.Item1; - switch (contract_cat) { - case ContractUtils.ContractCategory.Symbol: { + switch (contract_cat) + { + case ContractUtils.ContractCategory.Symbol: + { ViewStream res = get_sqlite_str(tup.Item2, true); mySetRealSource(res); // this creates the continous stream for 1 min bars, but we store it in the DB, so we just read it from it @@ -42,7 +51,8 @@ namespace bts.BlueOrb { // mySetRealSource(res); break; } - default: { + default: + { ViewStream sql_str = get_sqlite_str(tup.Item2, false); ViewStream rs = new FilterStream(sql_str, $"symbol='{symbol_or_contract}'"); // filter here @@ -158,7 +168,7 @@ namespace bts.BlueOrb { } private Source _sourse = Source.SQL; // get from config - private readonly string _location = "C:\\Users\\m_tah\\development\\data\\"; + private readonly string _location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:\\BlueOrb_Data\\":Path.Combine(Environment.GetEnvironmentVariable("HOME") , "BlueOrb_Data"); private readonly string _db_name = "stream_data.db"; // the symbol is used ot retreive the correct table, the data flows as-is from the table // with some name / type changes diff --git a/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLToDSLStream.cs b/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLToDSLStream.cs index 395ada703..8863e199a 100644 --- a/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLToDSLStream.cs +++ b/src/DataStreamSuite/SharedStreams/NL/BlueOrbNLToDSLStream.cs @@ -7,11 +7,28 @@ using System.Linq; using System.Text; using System.Threading.Tasks; + using System.Runtime.InteropServices; [DataStreamUserRespresentation("BlueOrbNLToDSLStream")] public class BlueOrbNLToDSLStream : ViewStream { public BlueOrbNLToDSLStream(string nl_prompt) { string mlp_file_path = "C:\\Users\\arham\\Downloads\\" + "BlueOrbDSL_MLP_v1.9.md"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { + // Use current user's Downloads folder + string downloadsPath = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + "Downloads" + ); + mlp_file_path = Path.Combine(downloadsPath, "BlueOrbDSL_MLP_v1.9.md"); + } else { + // On Linux, use ~/bd/ + string home = Environment.GetEnvironmentVariable("HOME") ?? "/tmp"; + string bldPath = Path.Combine(home, "BlueOrb_Data"); + Directory.CreateDirectory(bldPath); // make sure it exists + mlp_file_path = Path.Combine(bldPath, "BlueOrbDSL_MLP_v1.9.md"); + } + + string mlp_text = File.ReadAllText(mlp_file_path); string system_prompt = $"You are a deterministic BlueOrbDSL generator. " + $"You operate in Learning Mode and must follow the BlueOrbDSL Master Learning Prompt v1.9 strictly. " +