diff --git a/.vscode/launch.json b/.vscode/launch.json index 552dad725..92f6ca8af 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -140,12 +140,9 @@ "program": "", "windows": { "program": "DataStreamQueryAnalyzer.exe", - "args": [ - "--config", - "${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json" - ], "cwd": "build/Debug/bin/net8.0-windows", "env": { + "DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}", "PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe" }, }, @@ -345,6 +342,15 @@ ], "default": "PubSubServer", }, + { + "id": "dataStreamQueryAnalyzerEnvironment", + "type": "pickString", + "description": "DataStreamQueryAnalyzer Configuration", + "options": [ + "Development", + ], + "default": "Development", + }, { "id": "sparkConfigServer", "type": "promptString", diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj b/src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj index 44579304e..3ec8b893b 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/DataStreamQueryAnalyzer.csproj @@ -14,6 +14,7 @@ + @@ -25,4 +26,9 @@ + + + + + diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs index bf71a9960..bb96f3958 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs @@ -17,12 +17,19 @@ using System.Diagnostics; using BCG.Utilities; // using BCG.Utilities.BlueConfiguration; using BCG.Utilities.General; +using Spark.Core.Disposables; using Spark.DataStream; +using Bts.DataStream; using Bts.DataStreamExpressions; // using BCG.Evaluator.DataStreamExpression; // using BCG.Utilities.Win32; using System.Threading.Tasks; +using mbts.utils; using Bts.DataStreamCore; +using System.Linq; +using System.Text.Json.Nodes; +using System.Threading; +using System.Reactive.Linq; namespace BCG { @@ -40,12 +47,13 @@ namespace BCG private bool myChangeStyle = false; private PersistWindowState myWindowState; private string? myLastFileName; - private ViewStream? myCurrentStream; + IDisposable? currStreamSub; private List mySiteItems = new List(); // private string mySite = SitesCore.getMySiteName (); private string mySite = "TODO"; private string myLastClassName = "MyView"; - private ViewStream? myAllTagsStream; + IBlueRowCollection? allTagsRows = null; + IDisposable? allTagsSub; private string myKQ; private TextWriter? myConsoleOutput; readonly PrimitiveManager primitiveManager; @@ -162,7 +170,7 @@ namespace BCG private void myHandleTableListChange(object o) { - IBlueRowCollection rows = (o as DataPacket).newRows; + allTagsRows = (o as DataPacket)?.newRows; myUpdateTableList(); } @@ -176,6 +184,11 @@ namespace BCG #endregion private Stopwatch _stopwatch = new Stopwatch(); + object locker = new(); + (IDataPacket DataPacket, long Version)? pendingData = null; + long pendingVersion = 0; + bool isBusy = false; + private void myTryExpression(string expStr) { _stopwatch.Restart(); @@ -193,14 +206,41 @@ namespace BCG myErrorTextBox.Text = ""; // myDataGrid.CaptionText = ""; - if (myCurrentStream != null) - myCurrentStream.onChange -= myCallback; - myCurrentStream = null; + Disposable.dispose(ref currStreamSub); myExp = exp; myCrono.Restart(); - (myCurrentStream = myExp.getView(new Dictionary(0), streamFactory)).onChange += myCallback; + // object locker = new(); + // IDataPacket? pendingDataPacket = null; + // bool isBusy = false; + var version = Interlocked.Increment(ref pendingVersion); + currStreamSub = myExp.getView(new Dictionary(0), streamFactory) + ?.AsObservable() + .Sample(TimeSpan.FromMilliseconds(333)) + .Subscribe(dp => { + lock (locker) { + pendingData = (dp, version); + if (isBusy) { + return; + } + isBusy = true; + } + // BeginInvoke(() => { + // while (true) { + // IDataPacket? currDp = null; + // lock (locker) { + // currDp = pendingDataPacket; + // pendingDataPacket = null; + // if (currDp is null) { + // isBusy = false; + // break; + // } + // } + // myCallback(currDp); + // } + // }); + }); myChangeStyle = true; //new SimpleBindableStream(myExp.getView(new Hashtable())).onChange += new DataStream.ChangeDelegate(myCallback); /* @@ -350,20 +390,12 @@ namespace BCG } // this is the stream callback - private void myCallback(object o) - { + private void myCallback(IDataPacket dp, bool resetGrid) { _stopwatch.Stop(); var timer = Stopwatch.StartNew(); - DataPacket dp = o as DataPacket; myCrono.Stop(); - Log.Ui.diag($"Received callback for {dp.schema.tableName}"); - if (o == null) - { - System.Windows.Forms.MessageBox.Show(this, "Received a Null Data packet", - "Invalid Data", MessageBoxButtons.OK, MessageBoxIcon.Error); - return; - } - Log.Ui.diag(dp.toIncrementalIDString()); + // Log.Ui.diag($"[dsqa_upd.json]{{ \"table\": \"{dp.Schema.tableName}\", \"added\": {string.Join(", ", dp.addedRows.Cast().Select(r => r.Id().ToString()))} {dp.GetIncrementalIdString()} }}"); + Log.Ui.diag($"[dsqa_upd.json]{dp.GetIncrementalIdString()}"); myToolStripStatusLabelTime.BackColor = Color.Black; myToolStripStatusLabelRows.BackColor = Color.Black; myToolStripStatusLabelTime.ForeColor = Color.White; @@ -374,13 +406,17 @@ namespace BCG Log.Ui.diag(Text); int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue; + // if (myDataGrid.DataSource is not SimpleBindableCollection currDataSource || resetGrid) { + // currDataSource = new SimpleBindableCollection(dp.newRows.schema); + // myDataGrid.DataSource = currDataSource; + // } + // currDataSource.Clear(); myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows); var force = false; - if (myChangeStyle && force) - { + if (myChangeStyle && force) { myChangeStyle = false; myDataGrid.Columns.Clear(); - myDataGrid.Columns.AddRange(Array.ConvertAll(dp.schema.columnInfo, _column_factory)); + myDataGrid.Columns.AddRange(Array.ConvertAll(dp.Schema.columnInfo, _column_factory)); } myCrono.Restart(); // myDataGrid.CaptionText = myCrono.time.TotalMilliseconds + " ms "; @@ -390,26 +426,38 @@ namespace BCG myDataGrid.HorizScrollBarValue = hscrollval; var varNames = new List(0); - var code = myExp?.print(9, varNames); - myPrintCode(dp.schema, varNames, code); + var code = myExp?.print(9, varNames) ?? string.Empty; + myPrintCode(dp.Schema, varNames, code); myConsoleRichTextBox.Clear(); // this was was very very slow //myConsoleRichTextBox.Text = BlueDataUtility.get_printed_rows(dp.newRows, (col) => 30); timer.Stop(); Log.Gen.info( - $"DSQA MainForm.myCallback table: {dp.schema.tableName} rows: {dp.newRows.Count} time: {timer.Elapsed.TotalMilliseconds:0.00}ms"); + $"DSQA MainForm.myCallback table: {dp.Schema.tableName} rows: {dp.newRows.Count} time: {timer.Elapsed.TotalMilliseconds:0.00}ms"); } private Stopwatch myCrono = new(); private DataStreamExpression? myExp; - private void myTimer_Tick(object sender, System.EventArgs e) - { + long lastProcessedVersion = -2; + private void myTimer_Tick(object sender, System.EventArgs e) { + IDataPacket? dp = null; + bool reset = false; + lock (locker) { + if (!pendingData.HasValue) { + return; + } + dp = pendingData.Value.DataPacket; + reset = pendingData.Value.Version != lastProcessedVersion; + lastProcessedVersion = pendingData.Value.Version; + pendingData = null; + } + myCallback(dp, reset); // try // { - ViewStreamDatabase.instance.doWork(); - MasterWorker.getInstance(false).doWork(); + // ViewStreamDatabase.instance.doWork(); + // MasterWorker.getInstance(false).doWork(); //} // catch (Exception exc) { // Log.Ui.warn(exc.ToString()); @@ -437,8 +485,8 @@ namespace BCG private void MainForm_Closed(object sender, System.EventArgs e) { - if (myExp != null) - myExp.getView(new Dictionary(0), primitiveManager).onChange -= myCallback; + Disposable.dispose(ref currStreamSub); + Disposable.dispose(ref allTagsSub); } private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) @@ -595,8 +643,9 @@ namespace BCG myLoadAssemblies(); - myAllTagsStream = myGetAllTagsStream(); - myAllTagsStream.onChange += myHandleTableListChange; + Disposable.dispose(ref allTagsSub); + allTagsSub = myGetAllTagsStream().AsObservable() + .Subscribe(obj => BeginInvoke(() => myHandleTableListChange(obj))); myTabControl.SelectedIndex = 3; myTreeView.AfterSelect += new TreeViewEventHandler(myHandleTreeNodeSelect); //do this lazily so we don't have to have bps, etc up to run dsqa @@ -614,13 +663,13 @@ namespace BCG m.Checked = false; mi.Checked = true; - myAllTagsStream.onChange -= myHandleTableListChange; + Disposable.dispose(ref allTagsSub); myTableListBox.Items.Clear(); // SitesCore.setMySiteName (mySite); - myAllTagsStream = myGetAllTagsStream(); - myAllTagsStream.onChange += myHandleTableListChange; + allTagsSub = myGetAllTagsStream().AsObservable() + .Subscribe(obj => BeginInvoke(() => myHandleTableListChange(obj))); myDataGrid.DataSource = null; myTabControl.SelectedIndex = 3; @@ -680,11 +729,11 @@ namespace BCG private void myToolStripMenuItemTest_Click(object sender, EventArgs e) { - if (myCurrentStream != null) - { - myCurrentStream.onChange -= myCallback; - myCurrentStream.onChange += myCallback; - } + // if (myCurrentStream != null) + // { + // myCurrentStream.onChange -= myCallback; + // myCurrentStream.onChange += myCallback; + // } } private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e) @@ -700,7 +749,7 @@ namespace BCG private void myUpdateTableList() { - IBlueRowCollection rows = myAllTagsStream.rows; + IBlueRowCollection? rows = allTagsRows; if (rows == null) return; diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs index b9e205ffc..cdd7748d1 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs @@ -1,54 +1,66 @@ using BCG; -using bts; -using bts.api.core; - +using Bts.DataStream.Services; using Bts.DataStreamCore; -using Bts.DataStreamQueryAnalyzer; -using Bts.PubSub; +using Bts.DataStreamCore.Services; +using Bts.Extensions.Hosting; +using Bts.PubSub.Services; +using Bts.SharedStreams.Services; -using CommandLine; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; -using mbts.config; +using Spark.Api.Services; +using Spark.Core.Services; using System; +using System.Linq; using System.Threading.Tasks; using System.Windows.Forms; static class Program { - [STAThread] - static async Task Main(string[] args) { - await Parser.Default.ParseArguments(args) - .MapResult(RunAsync, errs => Task.FromResult(1)); - - async Task RunAsync(DataStreamQueryAnalyzerConfig cfg) { - try { - cfg.MergeWithConfigFile(); - cfg.MergeWith(DataStreamQueryAnalyzerConfig.Default()); - PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub)); - await Configuration.InitializeAsync(cfg.Sparkd); - Log.Gen.info("Starting DataStreamQueryAnalyzer"); - var primitiveManager = new PrimitiveManager( - new PrimitiveManagerConfig() { - Assemblies = new() { - "DataStream.dll", - "SharedStreams.dll" - }, - }); - var mainForm = new MainForm(primitiveManager); - if (cfg.StreamFile is not null) { - mainForm.loadFile(cfg.StreamFile); - } - Application.Run(mainForm); - } catch (Exception ex) { - // Write the error to console first, in case the logging system is compromised. - var error_msg = $"DataStreamPublisher threw exception:\n{ex}"; - Console.Error.WriteLine(error_msg); - Console.Error.Flush(); - bts.Log.Gen.error(error_msg); - return 1; - } - return 0; + static (IServiceProvider Services, Task RunAsync) init(string[] args) { + var tcs = new TaskCompletionSource(); + var host = HostBuilderExtensions.CreateBtsBuilder(args) + .ConfigureServices(ConfigureServices) + .Build(); + RunQuiet(); + return (host.Services, tcs.Task); + async void RunQuiet() { + await Task.Delay(100); + await host.RunAsync(); + tcs.SetResult(); } + } + + [STAThread] + static async Task Main(string[] args) { + Console.Write("Starting DataStreamQueryAnalyzer"); + try { + var (services, runAsync) = init(args); + var streamFactoryList = services.GetServices().ToList(); + var streamFactory = streamFactoryList.Count == 1 + ? streamFactoryList[0] + : new CompositeDataStreamFactory(streamFactoryList); + var primitiveManager = streamFactoryList.OfType().Single(); + var win = new MainForm(primitiveManager, streamFactory); + Application.Run(win); + await runAsync; + } catch (Exception ex) { + Console.Error.Write($"DataStreamQueryAnalyzer crashed: {ex}"); + return 1; + } + return 0; } + + static void ConfigureServices(HostBuilderContext context, IServiceCollection services) => + services.AddSpark(context.Configuration.GetSection("Spark")) + .AddSparkApi(context.Configuration.GetSection("SparkApi")) + .AddMasterWorker(context.Configuration.GetSection("MasterWorker")) + .AddViewStreamDatabase() + .AddPubSubClient(context.Configuration.GetSection("PubSub")) + .AddPubTableBroker() + .AddRemoteQueryManager() + .AddPrimitiveStreams(context.Configuration.GetSection("PrimitiveStream")) + .AddMarketDataStreams(); } diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/appsettings.Development.json b/src/DataStreamSuite/DataStreamQueryAnalyzer/appsettings.Development.json new file mode 100644 index 000000000..19899ec21 --- /dev/null +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/appsettings.Development.json @@ -0,0 +1,27 @@ +{ + "SparkApi": { + "IsEnabled": true, + "Credentials": { + "User": "bts", + "Password": "" + } + }, + "PubSub": { + "Routing": { + "PublisherRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://127.0.0.1:22019", + "TagRegExp": ".*" + } + ], + "SubscriberRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://127.0.0.1:22019", + "TagRegExp": ".*" + } + ] + } + } +} diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/appsettings.json b/src/DataStreamSuite/DataStreamQueryAnalyzer/appsettings.json new file mode 100644 index 000000000..065abc5d9 --- /dev/null +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/appsettings.json @@ -0,0 +1,86 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "Spark": {}, + "SparkApi": { + "IsEnabled": false + }, + "MasterWorker": { + "Interval": "00:00:00.1" + }, + "ViewStreamDatabase": { + "Interval": "00:00:00.1" + }, + "PrimitiveStream": { + "Assemblies": [ + "DataStream.dll", + "SharedStreams.dll" + ], + "ExcludedTypes": [ + "SettlementPriceStream", + "OHLCVStream" + ] + }, + "PubSub": { + "HeartbeatInterval": 15000, + "MissingHeartbeatsBeforeGc": 50, + "DisposedConnectionCheckPeriod": 1000, + "ServerPort": 22019, + "Routing": { + "PublisherRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://pubsub:22019", + "TagRegExp": ".*" + } + ], + "SubscriberRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://pubsub:22019", + "TagRegExp": ".*" + } + ] + }, + "TagTypeToAssemblyMap": { + "Entries": [ + { + "HandlerClass": "BCG.PubSubServerTagHandler", + "Assembly": "DataStreamCore.dll" + } + ] + }, + "TagHandlerFactory": { + "Handlers": [ + { + "TagType": "DataStream", + "Assembly": "DataStreamCore", + "Class": "bts.PubsubServerStreamTagHandler" + }, + { + "TagType": "System_Sub", + "Assembly": "PubSub", + "Class": "BCG.SystemSubscriptionMapPublisher" + }, + { + "TagType": "System_Stats", + "Assembly": "PubSub", + "Class": "BCG.SystemStatsPublisher" + }, + { + "TagType": ".*", + "Assembly": "DataStreamCore", + "Class": "bts.PubsubServerGenericDataHandler" + } + ] + } + }, + "Database": { + "Path": "", + "Query": "" + } +}